{"meta":{"start_date":"2026-09-14","end_date":"2026-09-20","start_at":"2026-09-14T00:00:00Z","end_at":"2026-09-21T00:00:00Z","timezone":"UTC","generated_at":"2026-10-14T08:00:06Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":728,"failed_runs":607,"raw_occurrences":1212,"matched_occurrences":883,"runs_affected":278},"rows":[{"environment":"dev","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":191,"runs_affected":78,"failed_runs":78,"run_impact_percent":10.714285714285714,"anchor_occurrences":191,"after_last_push_count":62,"trend_counts":[0,0,0,71,97,19,4,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":78,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":12},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":8},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":13},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":11},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":9},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":7},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":10},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":9},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":9},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":16},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":11},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":13},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":8},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":14},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":14},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a06120\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wh7bztj94wh4\" to finish creating: GET https://rp.j8800640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9bd4a4e-b6a1-4ac5-a23e-ee09a38de2f2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9bd4a4e-b6a1-4ac5-a23e-ee09a38de2f2\",\n      \"name\": \"a9bd4a4e-b6a1-4ac5-a23e-ee09a38de2f2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-19T20:34:43.428610597Z\",\n      \"endTime\": \"2026-09-19T21:03:49.814289269Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 24m7s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a06120\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wh7bztj94wh4\" to finish creating: GET https://rp.j8800640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9bd4a4e-b6a1-4ac5-a23e-ee09a38de2f2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9bd4a4e-b6a1-4ac5-a23e-ee09a38de2f2\",\n      \"name\": \"a9bd4a4e-b6a1-4ac5-a23e-ee09a38de2f2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-19T20:34:43.428610597Z\",\n      \"endTime\": \"2026-09-19T21:03:49.814289269Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 24m7s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d24520\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fgvlnn5kdj25\" to finish creating: GET https://rp.j8800640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ec9c856-afdc-48ab-ad6a-4c93e8b9216a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ec9c856-afdc-48ab-ad6a-4c93e8b9216a\",\n      \"name\": \"8ec9c856-afdc-48ab-ad6a-4c93e8b9216a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-19T20:35:08.292814934Z\",\n      \"endTime\": \"2026-09-19T21:04:09.799274903Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 24m13s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d24520\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fgvlnn5kdj25\" to finish creating: GET https://rp.j8800640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ec9c856-afdc-48ab-ad6a-4c93e8b9216a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ec9c856-afdc-48ab-ad6a-4c93e8b9216a\",\n      \"name\": \"8ec9c856-afdc-48ab-ad6a-4c93e8b9216a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-19T20:35:08.292814934Z\",\n      \"endTime\": \"2026-09-19T21:04:09.799274903Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 24m13s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001da320\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-rpq79wd7j7md\" to finish creating: GET https://rp.j8800640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6e591bf2-2064-49c7-857a-322459bd75aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6e591bf2-2064-49c7-857a-322459bd75aa\",\n      \"name\": \"6e591bf2-2064-49c7-857a-322459bd75aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-19T20:34:05.40086812Z\",\n      \"endTime\": \"2026-09-19T21:03:09.741249314Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m39s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001da320\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-rpq79wd7j7md\" to finish creating: GET https://rp.j8800640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6e591bf2-2064-49c7-857a-322459bd75aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6e591bf2-2064-49c7-857a-322459bd75aa\",\n      \"name\": \"6e591bf2-2064-49c7-857a-322459bd75aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-19T20:34:05.40086812Z\",\n      \"endTime\": \"2026-09-19T21:03:09.741249314Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m39s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7024/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101401374418800640","occurred_at":"2026-09-19T20:01:44Z","signature_id":"9aab4f6f9d6cdbeed3649de3e1c08ba30976e2dfd7744b758bbe58b90a7d6642","pr_number":7024,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7024/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101401374418800640","occurred_at":"2026-09-19T20:01:44Z","signature_id":"d585d82db665c5540b4bd08b657fc9869f9ae854709b04f0419699db90dd8006","pr_number":7024,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7024/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101401374418800640","occurred_at":"2026-09-19T20:01:44Z","signature_id":"f5f9841b8f36437b2d7d98afe3badb8c07122743349b2cac64442f9868c76ce8","pr_number":7024,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7021/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101139510631862272","occurred_at":"2026-09-19T02:41:11Z","signature_id":"a8dc665dcc48006e229b0863a483f1a19dcd94761736b3feb05bb4b660fcdb30","pr_number":7021,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","signature_id":"99f885a8a863b5de08d9591137eee29d3c46e0095bea1b9e8046d833c161bfec","pr_number":7010,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"524cdf887bee162758b9f6eca5510d2578bb6ecf654e9948629c9452eaf1782b","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"65b5dbb9967304eedefd97b212163b3f8af9b0b2bb0188919a3d12f3a6538fc2","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","signature_id":"78800d7f9bd7ce60d82097401bbc1a1b346c5eb770d7eeedc635c6d48abac870","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","signature_id":"ba73d1a54ec8e3a0b02523dbd4276ff942ead7e6f9b4e52703889f225a290272","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100867050745368576","occurred_at":"2026-09-18T08:38:28Z","signature_id":"2e53b9a5382ed0f0493cfc0c48e9bf7aa226835370880aa8b0e3c984f3d43f8d","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100867050745368576","occurred_at":"2026-09-18T08:38:28Z","signature_id":"bb9a6198465bbe86bad7f0728097a5e8139953db856c6c8aad613b2d535d245f","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","signature_id":"001338be47cf8dcaa3fc3c500d6a6dec5a631054e1ca9f9d544c49d235b2dcb7","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","signature_id":"782ec114fda729c308336015b9ba3ea7dba7299080a2ca46beda618ff211a392","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","signature_id":"abcb5fca9b88f5de84bce4c63659906c7070748eea55c112da30810936b0fc97","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","signature_id":"82775d4e1d46d1566b3f4eab5619ed61b625107edb899f3a6d4e9d4864e726a1","pr_number":6984,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","signature_id":"925fc6ad3116b3a4af4d42c985323051e3b386c99d2bbff7d1917ed9c6b7ef67","pr_number":6984,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","signature_id":"a2d0482738306402a9aee374779b77d199fad57d181dc39a001fc3fa8cf50f3a","pr_number":6984,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","signature_id":"b838f89ca9ff0822f2ef510e9178d1d320d4033a5d5cd3ca5c98537110704efb","pr_number":6972,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780897212043264","occurred_at":"2026-09-18T02:56:11Z","signature_id":"c3a814dc2b1fc08b7b4fdb53010472ca317b4e82bd862888bcc25a0f8b4091ea","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","signature_id":"2e205df2fc76c59d7d5bced445a9af5fdbd8ffc0a4321fbfafdc448b6041ea0f","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","signature_id":"993ed1b03e808393e377627c70564f00855cc6cb2d99bd1882ef2f7b7c73270e","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","signature_id":"a44cbebfe0aee283a67f51149529d8530cd9c5797c013288bce57d67f21ed3fb","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780819697111040","occurred_at":"2026-09-18T02:55:52Z","signature_id":"a093a69ee54b8dd613584cc9b286724efd055bbbaf92d5a169de74315234601e","pr_number":7000,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721105009381376","occurred_at":"2026-09-17T22:58:35Z","signature_id":"d601eba3abc4f82457501b40b90cc5d7561cc3383a0ad74d5298ef8e79a87795","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100714303135944704","occurred_at":"2026-09-17T22:31:33Z","signature_id":"7468e248b9c77a11eebc376f236b7610c67fe482e15792bf83562366e5934d5f","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100713215490002944","occurred_at":"2026-09-17T22:27:14Z","signature_id":"b9f0cd575efa9d87456c559803742a6d89fb91600ce45815cecb4689bcc702f8","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100713215490002944","occurred_at":"2026-09-17T22:27:14Z","signature_id":"db652501caa7a4389b880b5afa48aab924fa2baf6e02836e6489573d8b826dbb","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"6dff3d6d762cfe52b458282a2d61a5824494a4b7fbf9f3aab1ba2e26d05698c5","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"7bfd2cf527b8d88bab96af9ab4da562292d265e26e2314dcdcf6bffc6b100ac5","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"d74cdb3a0399ea27cc6c2ee2e3e86b457bc6ff248822afc15a1f630792589716","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"e593b0bc62b406e1f219043a4233fe6d4b2427734c6df50023d50f3222509f40","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"0a1d46d442635db3b7061c82189b8035af60a48310a53ecc646e31620910a1cb","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"50d86109b369a86d21ce7f771945b06926a4c38d2d49433ad0d362386d5132f7","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"78fecc2305e80f61dc9910998cc7c318760f78b41aed63eb7f46b3ce837e0d9f","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"aef26cedd47d167588772783661deaa26628c5b57bcb923a1a0a520ea749663a","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"b07610b43263bb7e953aa2613bf43c66ef66f67a6d400930f3acc7d9b2871dfc","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6998/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685757428535296","occurred_at":"2026-09-17T20:38:07Z","signature_id":"f8f97e2decd1a3d48fb001cc1c33150efca6eca10d42b34bfc0118d3f5dd19c8","pr_number":6998,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685654236073984","occurred_at":"2026-09-17T20:37:43Z","signature_id":"68698c4e2d81e96a811614a8eb209ad22dc4f07b9b07bee4f975ea46939d038d","pr_number":7000,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569791986143232","occurred_at":"2026-09-17T12:57:19Z","signature_id":"62a250c534c20b5ad145d249c89f89739cbae0c9b5704a8277cba737ff1f0047","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569791986143232","occurred_at":"2026-09-17T12:57:19Z","signature_id":"824964c0c575d45ef044ea907156d5bdd2f98d69978b43f2f1fb3b622414875f","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569791986143232","occurred_at":"2026-09-17T12:57:19Z","signature_id":"f24c215311a297177eaa94e5511fcc51ba57d864ab61313b4419552b1cf74930","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569759195074560","occurred_at":"2026-09-17T12:57:11Z","signature_id":"903f3f622deb52a1f1d8dea7e418d3ea7d8db5819b70eccb79ae8ea0a5299dd4","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100566804978995200","occurred_at":"2026-09-17T12:45:27Z","signature_id":"1fb4041dc59d81c7e183948830ae7ac1b2783b718bf3d7631b04090e55e3f847","pr_number":6886,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100566804978995200","occurred_at":"2026-09-17T12:45:27Z","signature_id":"3a922f4d19da47f001e6311e71c0f1a6d255911648780e0fdc7768301e19e9b8","pr_number":6886,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100566804978995200","occurred_at":"2026-09-17T12:45:27Z","signature_id":"e828cec50a4c240181974579c6679994f89f3d798f5316edd2d635df29854690","pr_number":6886,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100563400982532096","occurred_at":"2026-09-17T12:31:48Z","signature_id":"9771d1687cb82ab4b14c0f80173555cf109826201233928e90423e0171bc2724","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"85532b20bf57446a7d41586b4163ec6f19b5563d86b6207d2b7d5ab6951e2651","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"ab6e130ec463d100706975248c724bd66e5d7d3642c918a3434eeabaf4b5aa60","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"b8f768f8fc135387ca0e0f20dc96f1cad4999bade1e76fff9ca2066634cf47d3","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"cb6c78925476f970ec41ca2cc86883f3fe25eedebef129a57d35868eedaf968c","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"ff4a343919b36fa3670725111608ea3b35a12089dab63a806e3c5e121a1fbb9a","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"236cb0e5207b1956ffac77c17fe63e39c6df7e6a37c02c56004d161df8a008ca","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"39a1e24d4800633f0d0b42d4f18189154efcf362e824edcdb07cb8b8b0dd68d9","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"401c9dc75b4b5ec84bc7eacc4dd5a3649404e5b9bb0cad164dff2ac11981d6c4","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"4dab39d1b1d724fed25dc69d84d189cab2ac6a8152b77e555635e3621d133e55","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"bb7a8e5fa783928e378dcc846c79e38f12b390e8f34e30b5e6460dbc694c3af8","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"ebcbd7286e42fab84ff7ea3f9b915f4d4683f1309800106ed6220b22cd9d54e2","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6994/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100557383175507968","occurred_at":"2026-09-17T12:07:52Z","signature_id":"ee498f50c2b2ac982c301f4ac0cae3713723036b9fbee8e63fbb794dec859320","pr_number":6994,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554708664979456","occurred_at":"2026-09-17T11:57:12Z","signature_id":"d54234423202acb80a8cc7b79b53caff0b960878b2770d3f2b50c496f32d7589","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554708664979456","occurred_at":"2026-09-17T11:57:12Z","signature_id":"f4470bf38da6c976133b96af81d9ecd329780f2aa1f6bfa931cdb8bcafadf34f","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","signature_id":"8063e53d23cc71e09e9af725e9b15fa0500e6877a011fce15107ebe822270942","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","signature_id":"8cd2901e21933a26dee207d65687d26f76901c588c4dc077ff1cf8179eac46e9","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","signature_id":"c31f9f905858e3bef622b1331144e2d854b98bc4c9aa6c6098d97c66fd0a17ca","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554684329627648","occurred_at":"2026-09-17T11:56:52Z","signature_id":"9577b3508cc1b362f70a7592604f657ac43e72478f7b4c69c3b22190b77112b8","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"2cb0fa49d019c2b0a11abf2d3312cdc656ba3974fcf4c96aa2f18765ffe8dee0","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"61b54a22bdc6f8db1004cb461b9c1ea0ce8e7bb226818ce98016843218649c01","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"6989180271f1d05831c6e509c6cc427effaad2b85f3bbba286006443200fcb35","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"78cd64b525be75a4d9388fde337bd307c022a2d8ce21d0ffc33bf9360fed27a3","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"7cf6bbc9da68baad7e4462e2758e70a60f2061703be4a96f8198d3c3759e5683","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"969e77e201e0ea5c3383048290794e2481a2bc4e0d76aac6da2b54ff4ff6694c","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"a21f245f7a5f77e335623baa9b0ddbede34c06b521c1c7c4d490e127ce4037e7","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"cd184104708d45339d6bffb3c0990b1efb5eafdada1ae446f6ffcf492fc8f488","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"f0d309a90401bc7c464e4f86124fff842aa1f093303fd8bfa28b66ac92405fe2","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"fe2081edacd08521e3b4731165e95ed1b4f17a242e9810e3d19e5b07192590fb","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100537768756121600","occurred_at":"2026-09-17T10:50:04Z","signature_id":"0d5854936e81c77013558f96a12c534160fb9ce058180d02b89c4d9a3544856d","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100525488945500160","occurred_at":"2026-09-17T10:01:10Z","signature_id":"3596b17636b3dd9dc619bcf3bc7003efbdc9033c334b506d05c614ad30a25383","pr_number":6972,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100525488945500160","occurred_at":"2026-09-17T10:01:10Z","signature_id":"b8057328bdd22a04aeb32a86271770e8c39d4d2cd6eac75a438bd766471d440b","pr_number":6972,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524951428665344","occurred_at":"2026-09-17T09:59:04Z","signature_id":"8bdd74c61a93beb235e7d38100063c0ca3d930717abd8601b9cb4dac72149e51","pr_number":5724,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524951428665344","occurred_at":"2026-09-17T09:59:04Z","signature_id":"d6f32f2cee9bee5817a4bb9c6f37367a9340014362f5ecc9c490646d7eab987e","pr_number":5724,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","signature_id":"7cee4e5352bcdeee9af619ecd98980da8f5a54689aadb5ebc237dc1aa40f7036","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","signature_id":"837607adb29247003f557d4cb6ffdfdb8aeac843b72cf365de5b9d8cdce185b2","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","signature_id":"b097d1bd6e31b2848274482e1d375642d3af07f55f2c3adb043f6e076a9933e1","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","signature_id":"b662a9003e08c3ef5e2286107d4a3536bcba752aaf78844948d28b110eee221d","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100520057816748032","occurred_at":"2026-09-17T09:39:42Z","signature_id":"2cc7ca1e205a8dfaf045d2763125c668a7279c4261f08b6ddee57e833c36210a","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"2cf84524e6ccdd0a761441daa39b744a2ba3952626e9461b7d45c15b3bcfd754","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"3f28cae00bcc4e17acb25bb050d65aad4ea199a510b36c178d2d712889664cab","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"acc3649bd8151c983690e39ec8f25e1c5473a348f8d0898d6d79bd8563a49a87","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"dc4d8b8d1deffa568a3e69aa51d30ab552205c7e9c9f140c68e72f873ccbd501","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506297609752576","occurred_at":"2026-09-17T08:45:01Z","signature_id":"2ac60a748539f98f2c88eefc3d7d9d565740acd60ee833fc9852acdd1cfcf35e","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506297609752576","occurred_at":"2026-09-17T08:45:01Z","signature_id":"6be75bbc95c50cbac96f78528dfa07add64f7a96b8fe35fe1510e1615554ba67","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506297609752576","occurred_at":"2026-09-17T08:45:01Z","signature_id":"7f2b1821d50e299f9faaaf1ec8623b52220c8b011f2741480e4d05563a5a26c3","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495749199433728","occurred_at":"2026-09-17T08:03:06Z","signature_id":"706ec129f8f00b0fced3839fa11ea395190e7f746c1a75084493342ffbc32a92","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"1254becb25675dbd9ede114df6e32fd10e76cd84f176a807e3d5dc72b936cfbc","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"1f2d8a520d63c686030e3b1f47535fed1c6501ab4abdd19d36fb0c76229a1cb8","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"32d4bd52ce4e4bf4bbb0c374a620144f8370e122a59cb1cce2ba39c24a22fcac","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"71eb266b069aadc565cf2dfa5c38d0b00bec544a93f9accc8e29672f14a64cd5","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"833379ae7b454d610a7efbf91b7df6a563effdab3e7cab0f6ff3de3a51cd9ae1","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"88399b6110d57a9572b0144a071be4bf2ca4aeadc0f450fbc20ec223beef26bc","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"ced0fb0de9a8798a8f89849b4e849e1ad6b42de9776ac8d0c57c01c8648db002","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"ffd6f4632b90a496bd463d4995c3c420f976f2007427581611b1c7bcf4dd4b9b","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494074514509824","occurred_at":"2026-09-17T07:56:27Z","signature_id":"109feb9a36dbfe6e2dc23c6d08e79142806f38590e9a903022f2d16ea982a69f","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494069930135552","occurred_at":"2026-09-17T07:56:26Z","signature_id":"38588499aa79b332fce0da4da61a0607d4dd3683eb3c9cf92c9d624bfee5f925","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488796448493568","occurred_at":"2026-09-17T07:35:28Z","signature_id":"d26e879298c7f737b9ae16dbe160a711548fa3f84ab3c2a0048bc582a13e4076","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100479104343085056","occurred_at":"2026-09-17T06:56:58Z","signature_id":"008a91fd3e3e5f5195f74645a3769624ed2be2ab43fcea742cd06f90df27599f","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100479104343085056","occurred_at":"2026-09-17T06:56:58Z","signature_id":"6b862675f286ba82e4aaaeabbdde3ab8b1accc783182f06adb944ac909b9f88b","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100438339525021696","occurred_at":"2026-09-17T04:14:58Z","signature_id":"82d6e28574c7f34a07c333ca94f997752b96649f6b80a188942914d02804a6b8","pr_number":6986,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100435900084260864","occurred_at":"2026-09-17T04:05:17Z","signature_id":"2938ba362ea03badfa4fef3a5a8b455d8db3137be286b0a32f0e06cb64bc56e1","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100435900084260864","occurred_at":"2026-09-17T04:05:17Z","signature_id":"89c35af4f14189b65c64576cd7490a0088f1c8ab485b614bf43cd6ed4c19e683","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100420606720741376","occurred_at":"2026-09-17T03:04:31Z","signature_id":"72e9dcaaa03f26fa26de2141ffb6133bfeb9f62e98e7bbfc149c580a12c625ab","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100404999925272576","occurred_at":"2026-09-17T02:02:30Z","signature_id":"17d592d037b2f2a804c6494b323dd9c8826c7ee66667ea66a41a04ee3ac208fe","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","signature_id":"1c3549dc55bc6c387adb5760f50c6e243c8a43f7b39ed0bf467dc5a0511968bd","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","signature_id":"3287e14a7b0a164b12f072d969ca8183f1647cf1f1f00a66de841c81d64ef6e4","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","signature_id":"a5eed4f538e16bf946e6c1d7bf29b73fc650cf9bfa21cbd95eec591f73113f97","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"0d64bfff34a7c1f61cc366edc7ee533840e5653ed9071035a92b925eab037b24","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"2913423376b3c9240235607ee0d4c675506def5b909b1312a4bdad97516cd24d","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"428648ff0c0135e57b228a05c39d11591ece01ed750e930d86d83fe847be740f","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"49af2dca8e5069846af20cde9f9397824ce26e8ea2d79c52e7c7b30e8faed262","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"721faaab8ec90430d24300487ededdb0a93721318cf4132ab4ace8fde5518506","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"a2ee109ec25c2b87338aedcebe4c8cc584f5a143bdc97ee69a387447d2c549b6","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100385816462233600","occurred_at":"2026-09-17T00:46:16Z","signature_id":"21b15360431929bdb593a2107c3896a1ca85c63523d6f4c20410fb74f623d0a9","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100354655547559936","occurred_at":"2026-09-16T22:42:23Z","signature_id":"f2b94e7451cbc051fbf0de3b84883a92be81c7f11c25cc3c65ce003806a07176","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351996258160640","occurred_at":"2026-09-16T22:31:53Z","signature_id":"6606b623ae18c88ac8a9b2ec7e9a18026cd7fd696af00a64c201797796b23884","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100346022097063936","occurred_at":"2026-09-16T22:08:08Z","signature_id":"40918e34615df9a134d5bd8ce9ad3cec41e9fc04a925911878e36407f747a316","pr_number":4878,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100346022097063936","occurred_at":"2026-09-16T22:08:08Z","signature_id":"be79ce9f9e2a3c3c1a4622c360ca3519703f1be9bdd39e307bbb6ae587836434","pr_number":4878,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100344321164185600","occurred_at":"2026-09-16T22:01:23Z","signature_id":"00790225a1ec5d2f2111abfe85ea313169365d1041de24fbe4fa745473e8e330","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"0f792fe0731eefcb63f38f15bca9a1aa450cefba466c1d19cff0ae8ed18fb263","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"2698bc1670e697e5f02bc62eaeda2db20989111b43c6c7fcc9af92e68d70c247","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"3728e9e0a9b25fc2b96787f8795446de81019e99ef3c6a67f320e9809b7bdae3","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"50dab7e83b7442f97fb4586fd719c0b5802925b2f02cc972849692995a093dd0","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"79d660f1bacf21e2718e2b4460bbd751ebcedbde1fed3611a2e8b06be51c9b26","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"8d9c0886049dce2e52d531fad148d6ed84eeb2446b9625587b295b408c739bc3","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"a52a7332954a9dce3940383eb5861ec4596c82a180f58ab40695aaa3094465e8","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"fc5496eac86ff2a1981e1d0213c04f2b8c1f98c1155641ca8fa469251820f45a","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"12b8d48d4627f15facb797beba81c6d3c5baa8cff824ed1576ee1a132af8dff4","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"27af66bf8605d41a0eed60544affb038a78040a481edac1756ed6ae13ca1bce6","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"6d4635ee801aeb366b63d020c6eeb3ed381519a10bc1a02c8e00a5d443d6550e","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"e798e0c9ad920ab5c7e8728fb7d75c823add0086ed40ac0f3ccf60ed14187ffe","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100299874200195072","occurred_at":"2026-09-16T19:04:46Z","signature_id":"5264dd1fe40eb5bea2fb2d3c2020b90fcd8c767922fd709e8151a6988e36b5d7","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100294017282478080","occurred_at":"2026-09-16T18:41:29Z","signature_id":"5b5a6e4f2745283c541cfa867e89616d0e7a33b663b83015979c872710cb256e","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","signature_id":"131d9678ec571fa272cfc7362879effebfa9b5e73d92d8d4799c9e8c4dbac244","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100262131709513728","occurred_at":"2026-09-16T16:34:47Z","signature_id":"4d2c343c16f83436cc3986db89b356fdfb3e28de9ebaa9ceda587e5572a6ffc6","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100262131709513728","occurred_at":"2026-09-16T16:34:47Z","signature_id":"7516a34fe822a219e86822cf9a1b8d51e620d585523f3da6508b9fb567e9b286","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252533153861632","occurred_at":"2026-09-16T15:56:39Z","signature_id":"8fab9814450d28b4f592dfdb0cd8c2267f5fe8d7043a907d74c9a720713cb056","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252533153861632","occurred_at":"2026-09-16T15:56:39Z","signature_id":"d2849a0a17cc4329034ec17d0d9ec1983dfbaf5324ca580cd9753b318ae0a9bc","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"51205dcbc010a62ffa53522b05b81b83d3e49f95a801c1ca50c89cf5aa533563","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"7b40ea74a666a71f36b6120544e0bab8b70aaea87e5e9b851b3d3cee913ad9be","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"8553584f5882f3e24dad240bf62c9d51df1d078a5dc179dc15818be725a00b81","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"9bdd91449d29da7a6a1036ded0dff71e7cee7dca6d3b72ee155f7272dbc7ec9e","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"f0391e98a8d1c457ea51b54726e3c7af785fdf41b5e338fe271c88dbc805d911","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"52b66be1d91cf6aa4302f446e0cb6737cf15f6d2f47af03effef5a56fd561fbd","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"53f4dbcf0527ee85fd51e8c2f839babdd74e956b703ba4e02f72dc66ca32e8b9","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"593263a65b95e6f760aa7210a803a4f621c8333076597839a59808dbbf66d1f5","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"92ab9e016befa8294f948260d25c72e7a1d80889506782e1f2a4bb4fd849697e","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"de1bc95acf9d163b0a4c5d87ba6c29060b3bb4b653bb97edd817c695a7ef262b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"eb1186f89f1efbd9f2fb8485a07c81a2445ab57e2c0781fc90a1717593153377","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100228663373991936","occurred_at":"2026-09-16T14:21:48Z","signature_id":"27221439d7603c27de714a5d301e36e3ab9f94a39958b4fd4679cc00d417d51d","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100227428122103808","occurred_at":"2026-09-16T14:16:53Z","signature_id":"0240604425a3af254fe1ce6e815c04f07285897c6355ead3254c138853f560f1","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"416d3122655e04bc6a1e6cd87fa70c5433f7bca4d0212f66aecf79af52b1d0a5","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"42ea74c98dd160984e4c829f799e27d45bf2b5018f73c2238f76e9d8d6a3f703","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"4634c0871869430762696feaab7758bce3c5d6614cf0816c70b2c3b8b4d3ac0d","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"773504a23a339bedaebd473b20c28b68aef28fa8120eb503e3c2ada5990e4cf7","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"97a8d855655aa481b47156e39fc63195c1bf1486f4d7c41cbeb84f0001f75fe9","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"d1ada8baec3cfc5a8db7ecb4d82edd81b7f41fd13101da84520ce2bb5cd103d1","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"3b8fdb12977219a6ee2876ab164d9d979ba1367fe5e61c35ac3016ab87e10d51","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100213017273700352","occurred_at":"2026-09-16T13:19:37Z","signature_id":"1e90b34a567db73b55b0b0389a5bd30033829227bcd0152b9cc1ea0cbbb4ce6a","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100213017273700352","occurred_at":"2026-09-16T13:19:37Z","signature_id":"212391e614ece2195740bc51838047e93b2e7ba13e0b53f412f14fba63b492c3","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100196783199096832","occurred_at":"2026-09-16T12:15:07Z","signature_id":"c6e0c1f0e5cc2283be488e36d90c53cb7425625d072448ae938150a9f6555ec0","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100196783199096832","occurred_at":"2026-09-16T12:15:07Z","signature_id":"d49dc00210e1aa19acac371c4b3bbf0262ff2f7023f27f607ca12332c134c5be","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"3bf8a07ef9de6057d5469a6d5677df4989f7484e06e3a81c4e3df9504d3c4305","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"3ec1682204cd21163b91258f7fd79ed3ef2df2235e522890ff723136f72137a4","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"5d3daeaf2541170cf2048425db2ad55ef8977c82724f199e4513a26eb43628d7","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"cedab3b9f607c3ec7848c570f240cf2b35352afbc688b78a3a96a96f564deb7f","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","signature_id":"40e2d334636405f418ffb52c5061fbf30366656d327d9d8fc83b6e9d9afd9837","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","signature_id":"5b34bf27b7432e062ab59dae95c6e2c2ce5726c184df3b6d5751452cf7c1aac8","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","signature_id":"7fe3931d094682c8af6a0a3a715b07aac46854b46c2bbb9f6ad09b29f24ea4b6","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","signature_id":"c5a888e0079b8b0888371862a2ca292117eb1766fc5723aec9962ec739421dc6","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161384225443840","occurred_at":"2026-09-16T09:54:27Z","signature_id":"3fd557fa1356d64ec7477ff53929400119de4eda55d74d194fe4734c675f465b","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161384225443840","occurred_at":"2026-09-16T09:54:27Z","signature_id":"49e9dc4352795e52ee09109b3c354fc6854bce4fbeed1fea1be113f6a6209aa5","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100153419804184576","occurred_at":"2026-09-16T09:22:48Z","signature_id":"1df8582823fde891257155971a259af217e7163cb77525129dce1bd9c178bb48","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","signature_id":"5a06bfe3d19814f7c0a59d9e13f30fd4c6f1a6a7313da746fc7cd3c44a2f7939","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","signature_id":"5aa313310cb85b534b6ae9ece7e869257889473057f8f6e20312e1cf45aa795f","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","signature_id":"8fc547e896d5e55d734d3ddd67ff9ad51e9b0658292885c96f9396cffb7b85a2","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","signature_id":"b19cd1523bca64c96d563324ee6475a23c1199cf8856040f6afdad3121dbc86f","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","signature_id":"d15fe82d34504ca3d38710b46e1d54bc220f9fcfedc628bca6bbc3cb0cd4dd6a","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"3f36db93067dda2082a5ba2aa4539a20530e919b951302455f99df670738d8ff","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"94776e25785ac46d0a71172bc6722ed5b40d2bea8cb9370d89cdf09f1fa5c805","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"ab25cdf796ca0e23837f6fcc81a18f44c0e9d3bfb187f38beeb217788731b679","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"1f776ef61b45ee091fed66db65a127929ae7ae7228c6ef8a3304e727c23cbe5d","pr_number":6879,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"9f89db0ec4b8c3901dcf3ea1dec0354567233659a0ca459f22959ae94e89dbca","pr_number":6879,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100119365197238272","occurred_at":"2026-09-16T07:07:29Z","signature_id":"8485b82a0c9f334d47923eba74cfc4cebce6914810909ffc70f3acbe8579b2bf","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100119365197238272","occurred_at":"2026-09-16T07:07:29Z","signature_id":"f6b4ca993a39a3f8a63f6cce0ebaa94328998c8c08a83ab0da84bb108e61207b","pr_number":6939,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":84,"runs_affected":84,"failed_runs":84,"run_impact_percent":11.538461538461538,"anchor_occurrences":84,"after_last_push_count":42,"trend_counts":[0,0,0,30,31,15,4,4],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":84,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":84}],"full_error_samples":["alert fired 24 time(s) (12 unknown, 12 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T00:36:05Z\n  Ended:       2026-09-21T01:14:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T00:37:08Z\n  Ended:       2026-09-21T01:15:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-21T00:37:08Z\n  Ended:       2026-09-21T01:19:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-21T00:39:06Z\n  Ended:       2026-09-21T01:19:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-21T00:39:06Z\n  Ended:       2026-09-21T01:18:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-21T00:39:06Z\n  Ended:       2026-09-21T01:18:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-21T00:39:06Z\n  Ended:       2026-09-21T01:19:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-21T00:40:06Z\n  Ended:       2026-09-21T01:18:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-21T00:40:06Z\n  Ended:       2026-09-21T01:15:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-21T00:40:06Z\n  Ended:       2026-09-21T01:19:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-21T00:41:05Z\n  Ended:       2026-09-21T01:17:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-21T00:41:05Z\n  Ended:       2026-09-21T01:18:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/etcd-0 has been in a non-ready state for longer than 5 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T00:24:26Z\n  Ended:       2026-09-21T00:30:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7913600-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2ggt01co6d2ehgcrfpefiv7co64vf-v20260901\", pod=\"router-fbcbc989-n4pcs\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv2ggt01co6d2ehgcrfpefiv7co64vf-v20260901/router-fbcbc989-n4pcs has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T00:24:27Z\n  Ended:       2026-09-21T00:30:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7913600-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2ggpos1dbpnrhtfim3df37oqp4d6v-q4q2f0t2i0p0l7c\", pod=\"router-cb77f855-wpm9t\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sv2ggpos1dbpnrhtfim3df37oqp4d6v-q4q2f0t2i0p0l7c/router-cb77f855-wpm9t has been in a non-ready state for longer than 5 minutes.","alert fired 20 time(s) (10 unknown, 10 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-20T17:59:48Z\n  Ended:       2026-09-20T18:34:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-20T17:59:48Z\n  Ended:       2026-09-20T18:34:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-20T17:59:48Z\n  Ended:       2026-09-20T18:34:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-20T17:59:48Z\n  Ended:       2026-09-20T18:34:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-20T18:00:47Z\n  Ended:       2026-09-20T18:34:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-20T18:01:48Z\n  Ended:       2026-09-20T18:34:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-20T18:02:57Z\n  Ended:       2026-09-20T18:34:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-20T18:02:57Z\n  Ended:       2026-09-20T18:34:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-20T18:02:57Z\n  Ended:       2026-09-20T18:34:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-20T18:02:57Z\n  Ended:       2026-09-20T18:34:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y/etcd-0 has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"911f13dc127a9ace065557f3d9d6b1c823b667662d3dabe8b6a218f33ca1bf47","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101809250517913600","occurred_at":"2026-09-20T23:02:29Z","signature_id":"7d106244608ace8de6f9cae03dbed9bf2952c22134ad7882a0e61735843e27ac","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"5e756453c54bd40ab2d5dd229f82f628f7919ef41eabbc02f3d959659f4f7234","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101683713959006208","occurred_at":"2026-09-20T14:43:39Z","signature_id":"19e557aaac4e676680853d6de094c41a779d1dca118ed39692cb9878dfd677f7","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7024/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101401374418800640","occurred_at":"2026-09-19T20:01:44Z","signature_id":"9a0cafa790584b536f05ed01ad1acd2654a09216502c01822a7fcf0ed85b2352","pr_number":7024,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7024/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101365667302215680","occurred_at":"2026-09-19T17:39:51Z","signature_id":"3f3bcbd58ca8944fd2bc4fc03c1749413e603be46d31969f5b38919923676b90","pr_number":7024,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101360030631596032","occurred_at":"2026-09-19T17:17:27Z","signature_id":"5ad93809d6b2cd48be6680ec9539485b6fde9555e1a8dba2b07effe96bdef747","pr_number":7019,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7021/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101139510631862272","occurred_at":"2026-09-19T02:41:11Z","signature_id":"50753a4888c147e0373aa944bdd9b16954d312678f43e0b061b0434b6147765b","pr_number":7021,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023746776109056","occurred_at":"2026-09-18T19:01:10Z","signature_id":"4beb77cb7f662a3ba0c82df6677c24959b741d146512bce1d7d72b33bd6e9a0e","pr_number":7010,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","signature_id":"adf91e76f1e8a11039895567a05fb0e5c3f1d56a9565f8cf0e71b7f1f6e5d0ce","pr_number":7010,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"991b7eb9b2760771a64ba10515aeb5d1b17b39f0d0d776abbc32c3f1fb85abb3","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100887669335658496","occurred_at":"2026-09-18T10:00:26Z","signature_id":"c3bce50243ff744e90a9975f976bf7851614b19682c44e007ffa00d7c6f101f9","pr_number":6999,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100886446859620352","occurred_at":"2026-09-18T09:55:36Z","signature_id":"fcdb5ce6d5eb5426a483be7faf7e19f591fbf253b3ebf7b8ca13d6e2fd0d11f4","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100885599773790208","occurred_at":"2026-09-18T09:52:14Z","signature_id":"002a8aff34d1219bae02653dd6f6335a032c8c53538dc0e88e47e06a4653de19","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100877868748771328","occurred_at":"2026-09-18T09:21:30Z","signature_id":"7eefe8fce4efeddab6a7bddfd5f7e5cb50289671f56d6e25708c25b9305ce796","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","signature_id":"be273c1646946db878b565bca63154b691913899e238a33aee949d4d46ae8e4f","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100867050745368576","occurred_at":"2026-09-18T08:38:28Z","signature_id":"d6db3f97bdaa9f3b8874f62b7846be7bdd694b5e229a6dfe93f7bd212a0dcb02","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","signature_id":"f47f0b6edf615e207ea20e3174c9c4b6c3860a8cd322039375d46696bfbe5705","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841559414018048","occurred_at":"2026-09-18T06:57:14Z","signature_id":"7b986d6b9e982cdf65d80f902e9563f9d1125a3f54f3c1a12116db416ae5f10d","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827581187100672","occurred_at":"2026-09-18T06:01:39Z","signature_id":"56136cc3eeacd2663934d686bb2d9b29fb8b76aaa1ea7443d05087ccb6f12b10","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100812371122458624","occurred_at":"2026-09-18T05:01:15Z","signature_id":"e85df63033962c2849bfc318fc85b0a8cba9948dc2466a5fa45b6b2724994bfb","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","signature_id":"4750aa456ebf384aa187370c588c7897567d97246241f2ecee3733d106d2cd85","pr_number":6984,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780897212043264","occurred_at":"2026-09-18T02:56:11Z","signature_id":"a0cd1baa14d1b2c8e6eaa3868608b05d0234717e1c141c4d01a43194b1743c96","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721105009381376","occurred_at":"2026-09-17T22:58:35Z","signature_id":"82e0a12c7cd95fd907f63543eeac97bdeae8d5f9e5a424bdde230db4ef5d4c57","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"1033d58df80ee9a7d767bdc19425ad417e226db41f4f4ad1a7aa6ed4d2688d8a","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100699515785842688","occurred_at":"2026-09-17T21:32:45Z","signature_id":"18121697c147e2ac50157b93c7f4175f042996dac56448b883bc040150ef3bb5","pr_number":7002,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"d2aa839fc01483bc5381babfe51639f5c2325311fddfd8c5d39caa32de46dc09","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6998/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685757428535296","occurred_at":"2026-09-17T20:38:07Z","signature_id":"927ffe3fc5118bce586a8a07b93ed80685d7fe4d247724a208069edabb849e6f","pr_number":6998,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569759195074560","occurred_at":"2026-09-17T12:57:11Z","signature_id":"2b92ea2834b8405b970ceee37d04d32b047f88649536fb5e87c255a7cf6b9715","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100566804978995200","occurred_at":"2026-09-17T12:45:27Z","signature_id":"4815c5ee092aaeb9f17a4f179de41c68c528994add0634df56951333903777fc","pr_number":6886,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100563400982532096","occurred_at":"2026-09-17T12:31:48Z","signature_id":"ece77cd8e3e47a8b500cc23dba72bce96033933da8cfc561a7bfe5d93a950f77","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"e8c855a1e7025053c74a73b5e48c4e6e96debc9ba1c7d6b424796483466cdf6f","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"777817b1e387b1f4492402d2f5429c721df7c24785215c5257845216b59cfccd","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6994/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100557383175507968","occurred_at":"2026-09-17T12:07:52Z","signature_id":"b66eb0c3d6894149e8e5bee7c3f68c812916f17da4df7663f06be67bb5aedc35","pr_number":6994,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100556682735128576","occurred_at":"2026-09-17T12:05:14Z","signature_id":"8d0248f7d9c1bf72afe316ba52f0dec7aa9b6c443c3506cf8e8d1a57be0d1c1b","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","signature_id":"f10ddf6a0df1f5dbb5292b3e49c11fae37f0c18b3ec87fcd7444bc0f39779449","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554684329627648","occurred_at":"2026-09-17T11:56:52Z","signature_id":"e2639a0797c1c9a4a16eba734d19e610cbd443c3f33dd4bdd37f3d91b31ccf54","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"35bdd9c6de24d9640b64ae41e61a7affc59db651a49ce7a2fb22727cdcad6dc9","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100525488945500160","occurred_at":"2026-09-17T10:01:10Z","signature_id":"dff627a9d4028dda60dcb2cc608e3aeb26b1fb64501996026bbb8ff831d19570","pr_number":6972,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524896286150656","occurred_at":"2026-09-17T09:58:55Z","signature_id":"fd2143a50a59cd5afd961a3854a9a94e4c9bfcd849575bf08cd24c269273e2c6","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","signature_id":"39a62cd962057ec142de0e390236317a312acabc0d41e0544a266cdbecf8c0b4","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100523023395196928","occurred_at":"2026-09-17T09:51:23Z","signature_id":"360514ce5b258c6295b00ba9d2f1d4fd30911299b6a3d2dd1a60f05cf773940e","pr_number":5724,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100520057816748032","occurred_at":"2026-09-17T09:39:42Z","signature_id":"5472bf78238dce0fe2f94f0aff922f667d82ac0c02290e6e2cd57f8d35e3a169","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"a26f8c2c335980bba3f6e37f89e2b89d15ca7b79b6682f883c830ab2918e5b05","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100507818527297536","occurred_at":"2026-09-17T08:51:04Z","signature_id":"7e105769b349ece072f3c06670207d3d4017ec8870c2a43a5330028ceb5d52e3","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506297609752576","occurred_at":"2026-09-17T08:45:01Z","signature_id":"962f9008502054736ab6ef9e552851e993e6894bbd6022a1a131a064d8689aee","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495749199433728","occurred_at":"2026-09-17T08:03:06Z","signature_id":"d549df554d86a3fb7f1af2edc775ca43f25de1f5239534d866f56be388d7b3fb","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494074514509824","occurred_at":"2026-09-17T07:56:27Z","signature_id":"371eb10e8c7a8df40477f826f6a0c01232597b190d53940fdbe6a8b4c627ede1","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494069930135552","occurred_at":"2026-09-17T07:56:26Z","signature_id":"f82050e70a810fdc9a71c82fe836c8741b43edf42cceee0a76e6119d660eb530","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488796448493568","occurred_at":"2026-09-17T07:35:28Z","signature_id":"826198aaa6fd1e4924bf9a5846057c446813d101971e8e7ab1f42a73a8fd3ad8","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100463746039156736","occurred_at":"2026-09-17T05:55:56Z","signature_id":"b8571782c9062ff08f362067402808930639b058a5f971bedc267ccc71e53617","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"082cd49a16ae701dbb38b1b79f0d1ac12e5a9df37271b1daf757d49fb6b996f0","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","signature_id":"3ddbbbbfcbe35abb46fc03a012581caf1092f9cbcef18f864c7ff0cb4097904d","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100385816462233600","occurred_at":"2026-09-17T00:46:16Z","signature_id":"e6916222b7c43fadfb77f5030602e037edec83d0ef773a05b13d08de9f0efc19","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100354655547559936","occurred_at":"2026-09-16T22:42:23Z","signature_id":"b66349955715ed443f1362f9a218926c741bbba15a7053b8231a572f7bce0c04","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351996258160640","occurred_at":"2026-09-16T22:31:53Z","signature_id":"e2e746b818d8319297b0cfc1ce06bb87bc7386a6e269943bf13e78c9d13237ab","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351264033345536","occurred_at":"2026-09-16T22:22:06Z","signature_id":"db6b8a56aaf730fdd696fc547f30dac94889302e62174f2dd2508fcd332898b0","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100344321164185600","occurred_at":"2026-09-16T22:01:23Z","signature_id":"600a7515642ad65bbfc931d4dd2a57d1a3c95b9594a86121e772644509ea9e97","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100335879556960256","occurred_at":"2026-09-16T21:27:50Z","signature_id":"c599c77a6372b3affcde70e85dd51dd5124ab9de3cabd625edc84ba8a133bbfb","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100327783220645888","occurred_at":"2026-09-16T20:55:40Z","signature_id":"4c70e8e6dfdf51fe895bb78044f498f38b34d7b31ddb973e4ed72258e72ee568","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"d21baacf92b823fbe47d168aff240c5f90998a22314dd5cc365610b36ceba21b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"ee56a79fd88c61821c015922fd0060df48fa8ea6f29e86e41766eb43042fab32","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"d8457d1570ea90a26bcd98437f37670575aae28bffe0f986f9b004d2d9264040","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298749917007872","occurred_at":"2026-09-16T19:00:16Z","signature_id":"4fe55ebf2e3687f10b470f9b50ef984b7f154b6ecd6235b377b7abf35f5fe025","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298495381475328","occurred_at":"2026-09-16T18:59:17Z","signature_id":"ed3bf953568238e27f1b875c5f5f12b7c39936c7b777f8037f83dc0b1f8c2feb","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","signature_id":"e4b9c749f68fadead7aeb206b02a1f85e9ae2c2c7142b04d727f3089de1f8541","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100262131709513728","occurred_at":"2026-09-16T16:34:47Z","signature_id":"b7c6ba500fe6358c309a0b082682885bdc0c4016fb309f71ff96ef107142bcca","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"8a55cdb17b2e0960799afe76d087110a5ecf992561f62b085684e8010de63d22","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"d167cd058619d1445e1dc6c6ed93bc2e291c8bbbd9eba4b08df3c14423f77517","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"4778f344e9b25530d8b9156641a99f6b2838c230f7532738e3f69e081b9f7a7e","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"045a81b731fe6b760898ced74dc07c1acf462fd8638ae785f456d496a795842e","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100213017273700352","occurred_at":"2026-09-16T13:19:37Z","signature_id":"422af2d2e106cf074701d11ee92eac11656adc2f791fd92e942ac51d5ff857ef","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","signature_id":"aede2952cb21cd0c07ff4ac502b4109b4236290ff6f43397958c99820cf77924","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","signature_id":"3a6cb6146cbf8c095d871f7807f59ec635628a6cc72ba515bf71641eb6ad5bd1","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"7451ba3db591daddff6044c850b50a806f031b44b26c0a9cb44f8b82e28dd244","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","signature_id":"382d6e1a50a58344a999c679c7ef69f1f3c041161e556f0be9d14b621d84b09e","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100176649516486656","occurred_at":"2026-09-16T10:54:59Z","signature_id":"f201db6f99c06ca51a3a0a3934aae0846a6ea6ba393747b6f6edc6872fb99a72","pr_number":6952,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161384225443840","occurred_at":"2026-09-16T09:54:27Z","signature_id":"ae8593e2f93d516941e79280e7827e07a691be6004452673f6399cae8b22f618","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100153419804184576","occurred_at":"2026-09-16T09:22:48Z","signature_id":"3a248c85aeccd44a74b5e82dc19ae5115320308561bbe219e225440971c85f86","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"cf50089acb0846a23f15800cf9d1588e5a42034409429bcfeb7f8b4bdce19e86","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"74727d2f774b690907a06926c56a326fbc39e84388995b5a3c7eceec1a64f678","pr_number":6879,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6962/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132023019507712","occurred_at":"2026-09-16T07:57:47Z","signature_id":"22a7d0041fe9eca9fd2523ee31fb7da254fed5da06967c558019791f8f2e970a","pr_number":6962,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"05b02974c8e5aa95fe3326e72aadd1d9c24fabbae417684be2c2a88ea3d78009","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100124639266082816","occurred_at":"2026-09-16T07:28:27Z","signature_id":"1bb0a319d54a133be2e86c848056c08984c613c97a376359e9e2876ee27aa821","pr_number":6856,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":70,"runs_affected":29,"failed_runs":29,"run_impact_percent":3.9835164835164836,"anchor_occurrences":70,"after_last_push_count":20,"trend_counts":[0,0,0,22,19,1,1,27],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":29,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20260901preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":15},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00139c1f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-kctnzvxjhvcd\" to finish deleting: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d51f09c-5120-4516-b9ee-a66be3d59304\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d51f09c-5120-4516-b9ee-a66be3d59304\",\n      \"name\": \"3d51f09c-5120-4516-b9ee-a66be3d59304\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T01:02:07.101805352Z\",\n      \"endTime\": \"2026-09-21T01:26:13.044715763Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2i1qqe304gugno1vb23hj27er6drc still exists (deletion dispatched at 2026-09-21T01:02:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00139c1f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-kctnzvxjhvcd\" to finish deleting: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d51f09c-5120-4516-b9ee-a66be3d59304\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d51f09c-5120-4516-b9ee-a66be3d59304\",\n      \"name\": \"3d51f09c-5120-4516-b9ee-a66be3d59304\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T01:02:07.101805352Z\",\n      \"endTime\": \"2026-09-21T01:26:13.044715763Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2i1qqe304gugno1vb23hj27er6drc still exists (deletion dispatched at 2026-09-21T01:02:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001838090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-s2nb9c\" in resourcegroup=\"rg-zstream-upgrade-4-21-2wc8bnqpjvsw\" to finish deleting: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7690c3-5896-4508-ab34-c5babe08c719\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7690c3-5896-4508-ab34-c5babe08c719\",\n      \"name\": \"3e7690c3-5896-4508-ab34-c5babe08c719\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T00:42:41.012625795Z\",\n      \"endTime\": \"2026-09-21T01:06:41.448864721Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2hu8mbpcvgabffnu4e7eeji5v72ts still exists (deletion dispatched at 2026-09-21T00:42:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001838090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-s2nb9c\" in resourcegroup=\"rg-zstream-upgrade-4-21-2wc8bnqpjvsw\" to finish deleting: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7690c3-5896-4508-ab34-c5babe08c719\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7690c3-5896-4508-ab34-c5babe08c719\",\n      \"name\": \"3e7690c3-5896-4508-ab34-c5babe08c719\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T00:42:41.012625795Z\",\n      \"endTime\": \"2026-09-21T01:06:41.448864721Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2hu8mbpcvgabffnu4e7eeji5v72ts still exists (deletion dispatched at 2026-09-21T00:42:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00174e0c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k6sk5ztcb6st\" to finish deleting: GET https://rp.j7913600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2aeb64dc-0a85-4998-9985-55bcafbd5fc4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2aeb64dc-0a85-4998-9985-55bcafbd5fc4\",\n      \"name\": \"2aeb64dc-0a85-4998-9985-55bcafbd5fc4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T00:42:34.467872497Z\",\n      \"endTime\": \"2026-09-21T01:06:34.900855786Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2gfj01vbki6vfv5k1lnt4j81kjqnb still exists (deletion dispatched at 2026-09-21T00:42:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00174e0c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k6sk5ztcb6st\" to finish deleting: GET https://rp.j7913600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2aeb64dc-0a85-4998-9985-55bcafbd5fc4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2aeb64dc-0a85-4998-9985-55bcafbd5fc4\",\n      \"name\": \"2aeb64dc-0a85-4998-9985-55bcafbd5fc4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T00:42:34.467872497Z\",\n      \"endTime\": \"2026-09-21T01:06:34.900855786Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2gfj01vbki6vfv5k1lnt4j81kjqnb still exists (deletion dispatched at 2026-09-21T00:42:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"48429fa1e1b278e83c9c1235be2155e15cb94e91feb6a91189f07417aac2381b","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"ae6ec300b2e1b5ec89e4505b36ba0351611d82380af69c6bc2aa2a3220759883","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101809250517913600","occurred_at":"2026-09-20T23:02:29Z","signature_id":"e4aee9aff864d2f43ab51682459e47fa9b2962afca634816bc039aba490af5e3","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101767145149960192","occurred_at":"2026-09-20T20:15:04Z","signature_id":"55fec78f783ce8cf181e98d25b36b712ff6eab60ce5978dd2cbcd0aee43a639c","pr_number":7033,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101767145149960192","occurred_at":"2026-09-20T20:15:04Z","signature_id":"753848c3bc300eb5da8ad64aa08adc3a8b8dafee7fb505106b6f5cbf501ef653","pr_number":7033,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","signature_id":"1de82b4d07b3d460f0773709ee7a723eb163f891926727a6696a1f16b19d90b5","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","signature_id":"35c9e2930b67189ab544a31c2246d603785a353beb1727154337c02311b5ba68","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","signature_id":"6b62ee5741da84d36547bbd46f0d60714cb4fa8627bb19d966f92a3eba03e28a","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","signature_id":"6ecc7b9cabb4ec501df9af223046a31e6f46790bc97afea6159cc8363fead315","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","signature_id":"ae135f0135bd68e7f6075a32b2c72f3bdce2c27a9f86f02269bf9ec0d9c29b8b","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","signature_id":"5e8455d974423d74da7a2764a6de9171d767628d73b6837a9e7152a9dc31c031","pr_number":7030,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","signature_id":"94c88bfe25b6cc5073858901db97efc9bfdacd7a00fdfffa45d0abf7a6f10cff","pr_number":7030,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","signature_id":"ba4ec05dffbf7aeb4b673f4a633bc6bb64ed25fa0b09ea36f4467a08976b4236","pr_number":7030,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"6430a6d078ca6a6f372c5a118c728de07a27f33f7e2dd5e2fdf9a7c60c12d6f7","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"81d3fa9d4a49bf594db48e69cfb405302d6d567f873fa4a49e7a724e1f6a4553","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"db98f57638dcde7ae6924500ef05678fc197bea0143f095eee401504ac389cb8","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"66049a9e3aec5154c649a5163e0d3ee1db514ce22e0071326628d432e4a129e1","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101715422586867712","occurred_at":"2026-09-20T16:49:39Z","signature_id":"c01cceb55dbabd6acf2c61e2aecc8f6bbf9895f9ca156a63db239382788d9d1e","pr_number":7033,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101708991846420480","occurred_at":"2026-09-20T16:24:06Z","signature_id":"7a7b14873ebc7e928a8df3faa64a26b8a0c2ace4736d3d613f6a89c5b2c7ebba","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101708991846420480","occurred_at":"2026-09-20T16:24:06Z","signature_id":"f680e9c6bf353ff7e3737b28ae5ee7746498d1917182a5b8a67f475e211714a3","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101705684675465216","occurred_at":"2026-09-20T16:10:57Z","signature_id":"2eff4fde8ffd9f3247fb66660611f250c208eb3abb98228934a36763af0a55d7","pr_number":7030,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101705684675465216","occurred_at":"2026-09-20T16:10:57Z","signature_id":"b369fa1435784bfd120c7576bed94a6e4a39fcf725052260c677eef0e62863e5","pr_number":7030,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101670587393904640","occurred_at":"2026-09-20T13:51:29Z","signature_id":"10316e5a87c9726ae96606bd5b16607ddf7b0d400689c8e8c0e050115901ad56","pr_number":7026,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7029/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643457066962944","occurred_at":"2026-09-20T12:03:41Z","signature_id":"5fb2a2918cf998e33798a60ea71579a7a8d626d7c294f1e7d5559e93c8cb6517","pr_number":7029,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101634420027953152","occurred_at":"2026-09-20T11:27:46Z","signature_id":"b854729ec479cb89d9449d9a6d7d41e7ad976014ffc19decf951194f7d9ca05c","pr_number":7026,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610157367627776","occurred_at":"2026-09-20T09:51:22Z","signature_id":"d3228063fdb4db7077f94a323b61ddb40a6e50e0a6a4ce81d902c823eb177dc2","pr_number":7025,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7023/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101588887028633600","occurred_at":"2026-09-20T08:26:50Z","signature_id":"3a8359bef606bfcfc9ffd99f5dc07bf9b62993cb12fd5d570d46aa5683d99f29","pr_number":7023,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7017/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101291668018827264","occurred_at":"2026-09-19T12:45:48Z","signature_id":"f8bcff589b91860e60fd7630f5cb3ebe6c66deffe030fe41e25461b3440939a0","pr_number":7017,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100931988906053632","occurred_at":"2026-09-18T12:56:34Z","signature_id":"946ebd2cdc29e94011829e7ef21165a52d1801fd362344870c8d6a3e7c5269c4","pr_number":6999,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"1200ec6f2e462f6a4057f0b6736f389e0eb4918e85be52611ba1988ed4a6b80f","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"26a26ee01c1fe63e9854b87ab08eead6a6fc7c9019b032530af1bdbbab01d85b","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"36c7d0ac2608fdd9b180eb76cfcef49dd063a5ddca7a5e1dd0eb82655e85e2bc","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"397c03aebb8f9974cabd4c075afe35800b5602c55edd622b828e4ee997bc99ca","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"51e487e0e9fc1b87185142de83749a342d6962b63a6106533902cf6918e54d5e","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"54d2d9e8120866d9194ddacd546d77becc75e8a251ac250bfb9fb8db3e040d85","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"83cf0cf00f201ed62094249bde1cad4326fd870d9fd95f573466f70f995ca9f3","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"a7c143432b9aa1db66f17c395896aa0a31866970f3d2152b89df2ba59a93d938","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"d5580f5abed7b32fb1241b3c5a25421aa30df617d15c7a847acda8ee9363e8f9","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"f1c4779490fb4bdafbfc35cb730136fb51e8d2baae0539cf8ed6ed1617f9f5b5","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"f1d1ec88637d584406805321e304585733a1d0d5f1dc372e90627e8f47025e62","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"1d5a773360bc54998f805181631ae95d9d4f7e700834cf8b2a7758ff8a01a8b3","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"5ffc110d49d5553b43d54e9686ac2a42d55e70924452ef37b6b6d552874166f9","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"8f0ccca9b1ab0f62cbe8877698c2eb88ab3fdc916c573bc80063829aa9d361db","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"d478c40d48df905f6ee8e97dc60c2a82d7080ab4ce2d7fdcba0c3903ce1b257b","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"f1fb92d8ffbf4155d66435595ff699141fa92d0cac0f75c60e648373d685e73b","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100553657568202752","occurred_at":"2026-09-17T11:53:12Z","signature_id":"e9fcf1ca8ec5a4e5b455439c42c7c5bdd9023eaf9099b980ad11a8e344c2334d","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"554758499da2dbd5ca2e6de15b8c5c7a3ee81c45e3edf3c5d71f98a2d63ec2c6","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"ff7aa9ff9517f007abe64da89b86cee0dfefa327cb83d202da998d1607130a76","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100314659356676096","occurred_at":"2026-09-16T20:03:31Z","signature_id":"06e7cdde30cfbab4250217a65b741f57948c596ead351300b63debe1170a60a3","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"62f925cd8f107a148e561a08817704dd4da0dc08360a885c312c5776ffe6894d","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"e259668ae1165ff12f8afd3ff5be66f96028094b8d861c7cbdff5a5c0c31f019","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"460210ebabcf024c04114b174b6545f46b58b1c28d70303d86e3b1dce093c204","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"a552e8119646d9c743aa9a252024bde026b39e6cefa5828c2a635099d2355f2a","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"c81fbcddae76e3d46ce74bd7eeec6dbb65a70aa3f34997838af6880b8bc4b471","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"f38603c674107979a7e5113b087deeb85d932689f0a71354d02c3879c176e60d","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"09330ba162b6af089c34735c70eca9ddd91f6516659badb8a358d7c06fa7a1f1","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"4baa98b743ff58863c2c3f11ae6c7794d41c41f07100fe61766b0978f6746f19","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"6b6d7951973760ce40ada8345eab0fc3affb480bfb3af182b9b92a268d16b16b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"cf0c7c2f7324da55d7b02b3480eb3ca7ec6d10efdd79ac18d6876f5a4e4fc8b8","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"32832ce99401d97a958b7dcdfe20140118a267411f3e842c201974409fd354fb","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"47db8067bcef5f852d6be174031d8a4891925f676b5063754561044e04b33b85","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"6422616296d3a5a4112b796133cccebf1e701aef5be2522f1440d40d7f73f94f","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"e114928abafbdcce9fc1b5a0f9590defe570e1bf5ceed7138ffdb8bb94282914","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"e94f3f5e69e8e136647e4790e270a1391748aa6703a074a6fa38b2687beb9db4","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"ea4b0a4eb2c285656f848886af6bb6a27818c286a7b34ef24973fc4e223cf7a2","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222891458039808","occurred_at":"2026-09-16T13:58:52Z","signature_id":"1cb320d43390d047edfe367eca720f9ae752f792b2067b9a7d668b21e611b437","pr_number":6952,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"16a76a4ed4fe90d47f48a9657d0216e8d1f87acf39a8dad4d8ba5c30878fe100","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100188464501755904","occurred_at":"2026-09-16T11:42:04Z","signature_id":"139ea4eb9a07c176246f76b251911c3691a6cbbed5076e752b52b4042c281baf","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100188464501755904","occurred_at":"2026-09-16T11:42:04Z","signature_id":"85767451e4aa0503e59ddd278738b04f06b742245943e9225dbc7e3560e04b05","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100188464501755904","occurred_at":"2026-09-16T11:42:04Z","signature_id":"d4cf5fc8868b958b193d13f740ed514c1bae0dd9b8489c2b78271801c3f259d7","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":67,"runs_affected":67,"failed_runs":67,"run_impact_percent":9.203296703296703,"anchor_occurrences":67,"after_last_push_count":37,"trend_counts":[0,0,0,46,21,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":67,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":67}],"full_error_samples":["alert fired 7 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:00:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8t9zj\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-8t9zj (namespace arobit) on cluster ci01-j3118336-mgmt-1 has averaged 131.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:10:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-g6d4v\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-g6d4v (namespace arobit) on cluster ci01-j3118336-mgmt-2 has averaged 183.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:04:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7b547f7d8b-bz92h\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container mgmt-agent-controller in pod mgmt-agent-7b547f7d8b-bz92h (namespace mgmt-agent) on cluster ci01-j3118336-mgmt-1 has averaged 183.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:04:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-86c78bc956-rx5ct\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-86c78bc956-rx5ct (namespace clusters-service) on cluster ci01-j3118336-svc has averaged 829.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:08:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-vnkdq\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-vnkdq (namespace arobit) on cluster ci01-j3118336-mgmt-1 has averaged 166.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:09:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7b547f7d8b-pnvkw\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container mgmt-agent-controller in pod mgmt-agent-7b547f7d8b-pnvkw (namespace mgmt-agent) on cluster ci01-j3118336-mgmt-2 has averaged 166.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-17T08:59:26Z\n  Ended:       2026-09-17T09:17:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-86c78bc956-ljk5z\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-86c78bc956-ljk5z (namespace clusters-service) on cluster ci01-j3118336-svc has averaged 226.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","alert fired 3 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T08:17:39Z\n  Ended:       2026-09-17T08:27:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3085056-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-49zq5\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-49zq5 (namespace arobit) on cluster ci01-j3085056-svc has averaged 168.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T08:17:39Z\n  Ended:       2026-09-17T08:27:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3085056-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-d4b6fd9f9-p4w2z\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container aro-hcp-backend in pod aro-hcp-backend-d4b6fd9f9-p4w2z (namespace aro-hcp) on cluster ci01-j3085056-svc has averaged 377.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T08:17:39Z\n  Ended:       2026-09-17T08:27:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3085056-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-78587f7f-wtrr7\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-78587f7f-wtrr7 (namespace clusters-service) on cluster ci01-j3085056-svc has averaged 226.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","alert fired 4 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T07:47:54Z\n  Ended:       2026-09-17T07:53:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6077952-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-648575ffd8-pgfs2\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-648575ffd8-pgfs2 (namespace clusters-service) on cluster ci01-j6077952-svc has averaged 276.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T07:52:53Z\n  Ended:       2026-09-17T08:00:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6077952-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-x8286\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-x8286 (namespace arobit) on cluster ci01-j6077952-mgmt-1 has averaged 128.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T07:54:53Z\n  Ended:       2026-09-17T08:01:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6077952-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-p74sm\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-p74sm (namespace arobit) on cluster ci01-j6077952-mgmt-1 has averaged 122.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T08:17:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6077952-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-648575ffd8-526ht\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-648575ffd8-526ht (namespace clusters-service) on cluster ci01-j6077952-svc has averaged 185.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100485495183118336","occurred_at":"2026-09-17T07:22:21Z","signature_id":"f62c0a71de4ea91a4231cade3331db31d540a634ddef3ee99f2bb1c6a99266f1","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100479104343085056","occurred_at":"2026-09-17T06:56:58Z","signature_id":"aac5573ba90ecce2e869b424d24d0b2a557bdaf7fb68a59bdb1ca60dcfc91361","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100474592526077952","occurred_at":"2026-09-17T06:39:02Z","signature_id":"aa91dd60d940dac3239718e70007e00a82448bdb09061df222f973e65da4b8f6","pr_number":6952,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100470565717413888","occurred_at":"2026-09-17T06:23:02Z","signature_id":"036570c8237e20bff362fa84f18b2757ecf6b7478b115ced0b1b6d96b52ae4ba","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100470315283910656","occurred_at":"2026-09-17T06:22:02Z","signature_id":"34004eeb935c0bb4ce4ba308740f33575b771c563e1c3c5cbd660527dc370ef4","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100469063225446400","occurred_at":"2026-09-17T06:17:04Z","signature_id":"07f59a2e15e5179fe3fd25749784f754d222157d8b27384b9e1cd64842ae8768","pr_number":6986,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100463746039156736","occurred_at":"2026-09-17T05:55:56Z","signature_id":"aaa081b0ec7f2d8d4a7e0e90f29ef2c663585ebb663aa2ffcb4b659e94e0c5f2","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100445405488615424","occurred_at":"2026-09-17T04:43:03Z","signature_id":"98874527ffbf8450aeb4dca36d3c24d835d1bfc8669e736b7a62e3aeb588719f","pr_number":6952,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100440653564481536","occurred_at":"2026-09-17T04:24:10Z","signature_id":"98473ea7cdc506ea6ed06c90a329992858dfe045473eb058bcb45937743f020e","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100438339525021696","occurred_at":"2026-09-17T04:14:58Z","signature_id":"d43422fbdada1a940ecd84da27859063da487a9ed357cd0dce3b03a5e1093c47","pr_number":6986,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100435900084260864","occurred_at":"2026-09-17T04:05:17Z","signature_id":"a15a05f0f8abcad7e64ec9115ec84f6ac3e540a459d0e285796dd856a7a6caa3","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100434265043898368","occurred_at":"2026-09-17T03:58:47Z","signature_id":"3e9ecaaa55541e660e55a54f116bbcb93077443ad73a39702bb7f87aa56a43f5","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100420606720741376","occurred_at":"2026-09-17T03:04:31Z","signature_id":"8469f16144ba929df5b9d7bb8ddc370454c226da3349d92d91c2c32c02a33ba7","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100415215534346240","occurred_at":"2026-09-17T02:43:05Z","signature_id":"05d61646d01952c54a7da4864113e0532f1dbe45108ad4402418ae2eaaf5a2b6","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100404999925272576","occurred_at":"2026-09-17T02:02:30Z","signature_id":"030419f084e0ac9f8645ed83bbb93e54a5eee29ea545c7c537285ba77f7a19f7","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"09d8e2ae3fbdc5cec5986413598d96675056845bffdde0b6a135d4a6c2fcafa1","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100399662610518016","occurred_at":"2026-09-17T01:41:17Z","signature_id":"5a274f8276581e7a1744e1e3283d6f882212c3ec0a22900f40bdbce973372f71","pr_number":6009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"5eff94552133a60d52e39851d75edd9d10579e18460fb3418389d4051065a47d","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","signature_id":"f729f652c9b63e98893c72b91f3a1326eda28d14e829f566594109b7326b49cb","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"eeada1c413c3ca5fc2af1c6cbb216ce484bf15370729d3a7b144bcec22517b3c","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100385816462233600","occurred_at":"2026-09-17T00:46:16Z","signature_id":"a4314857c749da303216d314351149e0065b1d08354cb26b1fbedfc555fc6412","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100373409937494016","occurred_at":"2026-09-16T23:56:58Z","signature_id":"1e3921ef26cf400f3e3eaae679c2c7938f1ecf3228c5ec7fae0e28ef8fa1159f","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100354655547559936","occurred_at":"2026-09-16T22:42:23Z","signature_id":"940a5892f0bde8defb7629a002baedf9228b8f92a2cf58838e303561286c491d","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100353767223332864","occurred_at":"2026-09-16T22:38:55Z","signature_id":"e71be1b08c965527fccf228721850a63e0d7ab82463979308010bd46b7d5afcc","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351996258160640","occurred_at":"2026-09-16T22:31:53Z","signature_id":"fc7a5f2beeb914b7f9d5e98e516ffb3d0e395b2c10277b6a8a41faeeb997a062","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351264033345536","occurred_at":"2026-09-16T22:22:06Z","signature_id":"38cba13ec304a2d65c8ec3a2bb0373cae43840dfeeed1f7a55cc2d8c840f0488","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351223038218240","occurred_at":"2026-09-16T22:21:29Z","signature_id":"fbe220275de66dc82c17bd70379b1968bfca47f26279aaeb942365f7083c7aae","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100346022097063936","occurred_at":"2026-09-16T22:08:08Z","signature_id":"5098cde8f56024f9b94cf40f2a466897158f3f3994b796f3a58cc2d52fcb7504","pr_number":4878,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100344321164185600","occurred_at":"2026-09-16T22:01:23Z","signature_id":"f7080c270925219a311e60803887fccb7900c43032060412377f7a288ab7c269","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100343556559343616","occurred_at":"2026-09-16T21:58:20Z","signature_id":"180e6e60fd74213dba13f27436fee7ff492e29f8f76edd5be5c54b377947d936","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100335879556960256","occurred_at":"2026-09-16T21:27:50Z","signature_id":"71b22a11302a00be21922c71aa067e721f76475093f648235ac53a36e12bcb39","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100335557920952320","occurred_at":"2026-09-16T21:26:33Z","signature_id":"ceaa91f4efdcdb0d1c542d52b327737fcf8ae4ccbf48f0c17aab82acbcc99238","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100331448274784256","occurred_at":"2026-09-16T21:10:14Z","signature_id":"30e6e0d7b43f0480d6e10d35a84534dd048930cbdf424724d15cc0834916efb3","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100328454825185280","occurred_at":"2026-09-16T20:58:20Z","signature_id":"cfb1506b480d229252a933ef91e34400b8d02a4638f13cabb01dc1d4175a17b8","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100327783220645888","occurred_at":"2026-09-16T20:55:40Z","signature_id":"18e89a273ad3f5766b2dd6e52e34589ba49632a0e92e5d7b81de104647e14a35","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"29b55dd4d895c72d0d59b9865df950f870325198fd7e112f1858c956d6c3812a","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"4ef72abc8b9948fd9bead06d17d6310bcbc699657b6ee721fd16b3106ac79918","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100314659356676096","occurred_at":"2026-09-16T20:03:31Z","signature_id":"d4858edb57d6296d8d8b3f66a13945cd04dcb772e32b66ac68791e2b7d64dc15","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"078c270d3b43c225b1670f600e86beb8fac31540b45fed0edb0e8f2b88023a38","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6904/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100310890917138432","occurred_at":"2026-09-16T19:48:32Z","signature_id":"ba7f166d742007eec805103f097b0619e99ae0ecb99c10d11604e7456c1148b3","pr_number":6904,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309676229922816","occurred_at":"2026-09-16T19:43:43Z","signature_id":"d7f945bb6e41f7832c5eef4f84c2ae943bce6eff034300bf30a51c4a754e1cdc","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309343839719424","occurred_at":"2026-09-16T19:42:24Z","signature_id":"efa1057a04dfc40b6e037e249cb0dc21b2c09a49fd06c904c78795387f5897e8","pr_number":6403,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309325879709696","occurred_at":"2026-09-16T19:42:19Z","signature_id":"c35a6da3cb62c984e490fa20921e5205ae8f9b5d5f473a81f2bc4d92f07256ce","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309243423887360","occurred_at":"2026-09-16T19:42:00Z","signature_id":"4e635a5a55fd9c70e99f4fc90a819fcbe842ed4f80ffca35e85d712a75adda03","pr_number":4878,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100299874200195072","occurred_at":"2026-09-16T19:04:46Z","signature_id":"94b33034edb32ac378faab020acb7aa1cb586b97112ae6c5f8f64b5e0f39f1b3","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298749917007872","occurred_at":"2026-09-16T19:00:16Z","signature_id":"569b1e0aed53209c579e2ada482c82175262b769185ccf3a5bafd789625f23b0","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298527354654720","occurred_at":"2026-09-16T18:59:25Z","signature_id":"2515546ca4a2ed9cc66209f7f5a861f329da8544f35e1712e6bef48626b727f0","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298495381475328","occurred_at":"2026-09-16T18:59:17Z","signature_id":"b7bbff7c4bc8e38db57964016b3e5dd22cbf69711555869ce39fed0d057ef5fa","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100294017282478080","occurred_at":"2026-09-16T18:41:29Z","signature_id":"d6b769c42dfc7af863ab56e421478bc2130ed7b8030470653f093468c2ae9445","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100293442402783232","occurred_at":"2026-09-16T18:39:12Z","signature_id":"6d07e5dea3d8417306b6aad1d07607ec094c290a529075bc5ae0f6c507ae8dc7","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100285588732121088","occurred_at":"2026-09-16T18:08:00Z","signature_id":"3165e7829bfab81efada837f8a5a846a713826a1a775685756db96ad2a2e7a0b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100280971763912704","occurred_at":"2026-09-16T17:49:39Z","signature_id":"5241ddfc74d49c5ecad2616742ded97b4a5f9bf5fbeee0f248252a9d33303902","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100273649465430016","occurred_at":"2026-09-16T17:20:33Z","signature_id":"2aaecc7a8aa85a870eddb45d3a454574d39a17a2e1971a571ef303ebfa1b5dfa","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","signature_id":"4aa09e305fda5dad99f7aa547324f1cc9a5313a1ed785f8832369357abebfa2a","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","signature_id":"e80538656acc5e08b78c0da3cbda2f5f642ff686a707fd278f435ee95c791d0f","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100262131709513728","occurred_at":"2026-09-16T16:34:47Z","signature_id":"8bca23390b6e128f806583b6f43999dfe3e0891c3867f9b00f777f7b70551bad","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100260298874163200","occurred_at":"2026-09-16T16:27:30Z","signature_id":"641a1726bac71eedb2360eec9a405c0d1e826983dcd4ea335ba7c7dfa3cebe9d","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100260058020450304","occurred_at":"2026-09-16T16:26:33Z","signature_id":"9a105f0d73b76a92d17a2de5a3542ed76121642b109339640b150bdbd8718601","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100253677548736512","occurred_at":"2026-09-16T16:01:02Z","signature_id":"e26ae0379e9c2f78baac16193a817ee1d4bd6be52f5b22f001aad146994821f2","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100253157446651904","occurred_at":"2026-09-16T15:59:08Z","signature_id":"bb91f870f2d6651db04eb26377a6ab0f560e56353470294354ffb9ea8c0407f9","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252538870697984","occurred_at":"2026-09-16T15:56:40Z","signature_id":"984ec4e1000694cd92d63a32c0f7a6e5d7b7f677e37cfce55ad670237d8b5da0","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252533153861632","occurred_at":"2026-09-16T15:56:39Z","signature_id":"59e4de1f8e513f5cc08bd4e593ae29483ec56ee54e8e92faff2ba25b8713cc09","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"fde97b6bc2d22cd4f0aa0ec9116922422f6896f61d1b4b08310d88dfdf610931","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"5176305f0eeec8bbbaa6ff316c0143a9416bf749550533697343b8a801ca28cd","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6923/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100244030364323840","occurred_at":"2026-09-16T15:22:38Z","signature_id":"3af7515b2be64cbe2958e1955e72cb60cdb6b11e0681c1a634bbc8ed9f8e15d6","pr_number":6923,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100243112008880128","occurred_at":"2026-09-16T15:19:04Z","signature_id":"8cecf1b875c80d5aaad598063b3c383cfb7ef9d6225c392907d6632715156b7b","pr_number":6923,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6726/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100185438990897152","occurred_at":"2026-09-16T11:30:02Z","signature_id":"5d5dbfb417b6897cd24a2f1e6ad614604212fa121b9f81e032502d4c16e94ed3","pr_number":6726,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":37,"runs_affected":37,"failed_runs":37,"run_impact_percent":5.082417582417582,"anchor_occurrences":37,"after_last_push_count":20,"trend_counts":[0,0,0,8,7,11,6,5],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":37,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":37}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T00:18:44Z\n  Ended:       2026-09-21T00:50:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4251392-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4251392-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T00:42:36Z\n  Ended:       2026-09-21T00:48:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7913600-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7913600-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-20T21:05:56Z\n  Ended:       2026-09-20T21:11:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8645120-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8645120-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"b10599bdf4bc1308f14fce45589e487e1d0d085cf0ef4c4bb64b2cf8c30f79c9","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101809250517913600","occurred_at":"2026-09-20T23:02:29Z","signature_id":"f65d23db3fce211d34d0769e73e3f8f7acba651c0170fe2e1546f79d4a71b579","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","signature_id":"8f4619349028d8271e222f8d6571b2f090f7d7878ecb45ed7f028937641f406f","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","signature_id":"e7c54520064dc8897e885bf4790b5cdcc177d7e87c6cdbf61692ea5647d049d2","pr_number":7030,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101600364640866304","occurred_at":"2026-09-20T09:12:27Z","signature_id":"f3bf8d60286004f12d28b5f26936659fa6d1debd6a2443cbc6166170ce25bcb6","pr_number":7026,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7024/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101438997736722432","occurred_at":"2026-09-19T22:31:14Z","signature_id":"ceb0d4cdfb0aadf0ebe2582014421740131c3e005ee97b0cc48ab55277be8eba","pr_number":7024,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101401417230061568","occurred_at":"2026-09-19T20:01:54Z","signature_id":"cbd5d731cdc57ceac0d94946a2a2d55d8b2e820731b79fc6f2eab46bf941ed5b","pr_number":7019,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7023/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101397745473425408","occurred_at":"2026-09-19T19:47:19Z","signature_id":"7fc0b33301708234bf3f44e2763cc66711e73109a90147d379f44c2c2cfbb55b","pr_number":7023,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7021/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101282005600702464","occurred_at":"2026-09-19T12:07:24Z","signature_id":"c24c890cdf10f23ec6d0fc10ec64e8b80627a0013caa6cf7af4d3f3a976149d7","pr_number":7021,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101255238609539072","occurred_at":"2026-09-19T10:21:02Z","signature_id":"52d7b0c372ebff950f627f4f32e163eecad6bf0af11a1cf994694c990af5e65f","pr_number":6925,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7021/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101139510631862272","occurred_at":"2026-09-19T02:41:11Z","signature_id":"20356976c000efea50c376a17cded2c1d14cae208b71b4de8f1d4798c54ac402","pr_number":7021,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7017/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101084611508441088","occurred_at":"2026-09-18T23:03:02Z","signature_id":"86c45d947e908dd24da56e029e4fb3fb50ede5048c717aadd07439573c0a5e5a","pr_number":7017,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101036649826750464","occurred_at":"2026-09-18T19:52:27Z","signature_id":"f8ce7b6dbb00b186dd3690b8bd0cde21b1999ed29a53641a6363be8c9bc8013e","pr_number":6945,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","signature_id":"1522dde0b9bb5d85e613b7efec8820e9ced25e3c11ff2b7a1c60a5642910649a","pr_number":7010,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6698/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100984946473570304","occurred_at":"2026-09-18T16:10:25Z","signature_id":"f7df40a156b9e56e6a5ddf150c7ac6e35049cd21802df2d33f93003f72173db8","pr_number":6698,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100987718002544640","occurred_at":"2026-09-18T16:01:55Z","signature_id":"db6a9f6935fcb92fd2604ae71dc7832d2001b5cc7b8edb8be0b3faebabe391c0","pr_number":7002,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100937243521716224","occurred_at":"2026-09-18T13:17:26Z","signature_id":"d6f983561c2168d8dc1fb0bb24a4ce58dae5f6ea67b850648a086ef0cc8ced1e","pr_number":7002,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100885599773790208","occurred_at":"2026-09-18T09:52:14Z","signature_id":"4e84895295e21c9ae798ddae187f381094acb88d5bb19172060995a42efdfbcd","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841605895294976","occurred_at":"2026-09-18T06:57:25Z","signature_id":"0d95d8184561f593338e86b1c11b5ad3f6442e5263869a130eac56ca5e35b079","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841559414018048","occurred_at":"2026-09-18T06:57:14Z","signature_id":"b8ef7bc40a5e42cb1d9a25c2c491ed35aa8060746d13168cf09c7ed965a89066","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","signature_id":"49184b986d87df135df2fc3dcc64465876a9918af7c03bd147e1ea6126eccebf","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100779821045911552","occurred_at":"2026-09-18T02:51:54Z","signature_id":"04a95f9ed56a68aa95bb9d5ccdef76442ca4969785b15ebf86b48f429dfe945c","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100680641501728768","occurred_at":"2026-09-17T20:17:48Z","signature_id":"d9ede6c71c00bb7dbee4ddd3fb9aecba02c72aceca980c3acfa9826e45ff50ea","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100563400982532096","occurred_at":"2026-09-17T12:31:48Z","signature_id":"e9c0e585944f16bc753c7dace1676a24a15848f5a821759ef30ea4b61ef466bc","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554684329627648","occurred_at":"2026-09-17T11:56:52Z","signature_id":"8c32a70da3a734350e4252477cd5e4400a822058b8ff76cfef91cedb261d1712","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524951428665344","occurred_at":"2026-09-17T09:59:04Z","signature_id":"a438005342e6e45a592ab0e52281d7a0356c34cf7a20d84dc4af128f220b1b66","pr_number":5724,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524826916556800","occurred_at":"2026-09-17T09:58:39Z","signature_id":"125c7452f650c84b80c4227b5aa637ed38ecce323a860ceae503938b79787746","pr_number":6986,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"795144da2ef4d9ca390a9485406bde85898d43c117d008e0d213f31d6d5042d6","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488384702058496","occurred_at":"2026-09-17T07:33:50Z","signature_id":"77026658b7ddf2ead4aff5831cdfc9e1e01b461fa183028dccacbe6855fbe9a3","pr_number":6952,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100260058020450304","occurred_at":"2026-09-16T16:26:33Z","signature_id":"84adc99394ffdde58d8777dabe660ecf4888bc0533bab685e83dea23a8c7fc1a","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252533153861632","occurred_at":"2026-09-16T15:56:39Z","signature_id":"eb143e51cc4e677a9811a011abde7f9846dfb86171fd5089044d86a86bc3c30e","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"4891bae57c16737882a8247821ec7f6d4e17417de53801a0bcb72589d84d269e","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100228663373991936","occurred_at":"2026-09-16T14:21:48Z","signature_id":"af0482283bab09054590813e51480bea798a5743b7bcc33fa9c63b58d8645bbe","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"6113c0f8fe30ddb0cec1586532a7684c6dc0ec614d444f96448deb0f4ad3a170","pr_number":6879,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132038643290112","occurred_at":"2026-09-16T07:57:51Z","signature_id":"bf9a145d432cea73856e5e5cd0fb2d2856d4a87e3d9fdb2343a2f93ed338bb52","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6923/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100110113200148480","occurred_at":"2026-09-16T06:30:43Z","signature_id":"fac8088074452240e37582cff853c6a9cc2e040277fbf34b005056861933699a","pr_number":6923,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100104517457547264","occurred_at":"2026-09-16T06:08:29Z","signature_id":"5f54bcb248196b637bd98df597f0841d329c7e41ad40465254245b7458522f8c","pr_number":6894,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":34,"runs_affected":16,"failed_runs":16,"run_impact_percent":2.197802197802198,"anchor_occurrences":34,"after_last_push_count":30,"also_in":["PROD"],"trend_counts":[0,0,0,7,1,3,1,22],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":16,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa01c0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-rsgvv4vlx279\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\",\n      \"name\": \"f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T00:26:58.238504906Z\",\n      \"endTime\": \"2026-09-21T00:56:02.680818108Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa01c0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-rsgvv4vlx279\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\",\n      \"name\": \"f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T00:26:58.238504906Z\",\n      \"endTime\": \"2026-09-21T00:56:02.680818108Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006001c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nzxl7wns7x4f\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b2c0231-f920-4a27-986e-a3e4ef41850c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b2c0231-f920-4a27-986e-a3e4ef41850c\",\n      \"name\": \"5b2c0231-f920-4a27-986e-a3e4ef41850c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T00:27:09.310854698Z\",\n      \"endTime\": \"2026-09-21T00:56:12.748107548Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006001c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nzxl7wns7x4f\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b2c0231-f920-4a27-986e-a3e4ef41850c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b2c0231-f920-4a27-986e-a3e4ef41850c\",\n      \"name\": \"5b2c0231-f920-4a27-986e-a3e4ef41850c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T00:27:09.310854698Z\",\n      \"endTime\": \"2026-09-21T00:56:12.748107548Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001226220\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-ctqghmzsdxw5\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed9a0c72-7da1-479d-80fd-6a53427b28d7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed9a0c72-7da1-479d-80fd-6a53427b28d7\",\n      \"name\": \"ed9a0c72-7da1-479d-80fd-6a53427b28d7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T00:27:10.309010209Z\",\n      \"endTime\": \"2026-09-21T00:56:12.764905068Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001226220\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-ctqghmzsdxw5\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed9a0c72-7da1-479d-80fd-6a53427b28d7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed9a0c72-7da1-479d-80fd-6a53427b28d7\",\n      \"name\": \"ed9a0c72-7da1-479d-80fd-6a53427b28d7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T00:27:10.309010209Z\",\n      \"endTime\": \"2026-09-21T00:56:12.764905068Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"0a72c2fe9bb203f971f92d8e51c30ced5c7cc31b87aab0f0b5be2aa7c9918b6a","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"1d7e40ff9457b0a0fd1f510836392b2a61ea6d2ec70f11c154614ed442a91215","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"36da18cbc5bd0fcbf1bcfacf064639d80f3a1a5cf27962716b02e77a1bbfec85","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"3e0062883f6a7b1aa6861d60c4689b05572197b0ccbb7b68cf2a6bb313f310ed","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"4566340774f4d5644a077677c8965bc9db5a33855b406b923890c6903dd9241c","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"5f4073b27d81b6ba979875f303e0026e5d88b19010e94d8fbda6cdd43405762d","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"7423340e196d990fa25ef185f5f19b2603dd00ed381a561dc6f7f6a62639d096","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"86240fd1d6cc9a1cfec477b60417613aecc1fa5308bed9e0b396aa7862dc467a","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"8e12d0852cea0a6ef4c2f62124f08fedad1faab413533a1887ee7ffe0186f31d","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"d6906ab33d96cae1ef1353787751cc0c7d0fd1b9e2f1ba9fdecffdd4391cd20a","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"e126abbe3fd20bf154b2a5a83d9a532f73d8c15de2b02bdae2fe166758e207a1","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"fb228348ca1cd58b436b3544af73f3d69eabd197936f9379767b0fcf28b2b9e2","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"eac5055c826c77b923258da473791d4fdf5723c4be718ddb37bd909244aa07ad","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"0122ecc6da0d7f29528efc4c03aacfe2a95abd22053ee427b3ba034f89f24578","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"6bc7534690e6bb03e8cc5f05e1f2bfee87b13f961ed915d568dd8e6b92f0e0f2","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"7486390772235b7b4ad872f03600862b7205a72c98df5cd28e716390fedf267b","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"a2be2cf8041303505466850a3cb9c21d1f99bbb048308b395ef9642bc9197ca2","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"a682bdcc6ac5308107646bacc01ed06ff73822f3817c828a7af6eeb2d50dbb37","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"aea772e72e8fe85891b43563346768f4449021077f900a91ce91ffe2eb62aef3","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"bf5a3ba4ae1888ec5bbf4c1ea6088363a29b33944bb08384b179d0e7edc222cb","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"c4e33f88c11b927f53f19ef5f88b383cbdc6921ffa0d1b5ed16a016693d69ee1","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101673315566358528","occurred_at":"2026-09-20T14:02:20Z","signature_id":"21b7ce7b54d77195fc617e821329b90e750c10192c6be2fdd3f4089b6662713d","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101105796258467840","occurred_at":"2026-09-19T00:27:13Z","signature_id":"6c98fd545a05be7580986790f198987d59741043c314322d01ffb8cd903aac97","pr_number":7019,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","signature_id":"2c332da578489a812e0144aacf458767c886ed8769b126097a13ac0573609822","pr_number":7012,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100877868748771328","occurred_at":"2026-09-18T09:21:30Z","signature_id":"f590a6d9decf3d5d50d5dfcef242c5c8af6ebdbb758d3e90d6fae5136781bae3","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","signature_id":"89409b5febc6093a9cefc9b6ca32afd019b45b211957e3bd23fe77c3eda373f3","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100478808179085312","occurred_at":"2026-09-17T06:55:45Z","signature_id":"ebdc66289f4002396961893fc85f6cbf2d72c402de90efcaab065108518df75a","pr_number":6988,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"7037bf26441097d6041fad63a8ad90e3d7ac2c0bc29160794aafceabed76fa0d","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100293442402783232","occurred_at":"2026-09-16T18:39:12Z","signature_id":"3d2d19c49b473c9b2b5c1bd828e9238e2a0bdd4c4f8d89fe2be19a6ea937a4b7","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","signature_id":"4c86e6598dc5b904362415f87179a0f883f34b4b44e009896f6a7f0de46090e2","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"9ae40ec334e2759457083c312bb3ac6d6fef401546cf0c8751fc4d293a56e796","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","signature_id":"f792cb486e1be9a347881acf7c3d2cd96a120fd226d905b4a6c6fa830ffd406b","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6726/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100185438990897152","occurred_at":"2026-09-16T11:30:02Z","signature_id":"681376edcfb91afa0eef4e45ea02cce5668cf8a6c8cbfde456dd6c90a347a910","pr_number":6726,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132115432607744","occurred_at":"2026-09-16T07:58:09Z","signature_id":"76ed85811ec3a7dce6c999043c0cc9d5ea6c96b382b975abd6e4372f70524c64","pr_number":6964,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":33,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.27472527472527475,"anchor_occurrences":33,"after_last_push_count":26,"trend_counts":[0,0,0,7,26,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110a440\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qr9h99g92762\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2e04524-dc8f-4310-a549-fb6dc0199f5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2e04524-dc8f-4310-a549-fb6dc0199f5f\",\n      \"name\": \"c2e04524-dc8f-4310-a549-fb6dc0199f5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:28.114427313Z\",\n      \"endTime\": \"2026-09-17T02:31:33.96482384Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110a440\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qr9h99g92762\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2e04524-dc8f-4310-a549-fb6dc0199f5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2e04524-dc8f-4310-a549-fb6dc0199f5f\",\n      \"name\": \"c2e04524-dc8f-4310-a549-fb6dc0199f5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:28.114427313Z\",\n      \"endTime\": \"2026-09-17T02:31:33.96482384Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0aba0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-ms9kkl5qh4rm\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\",\n      \"name\": \"7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:38.567378599Z\",\n      \"endTime\": \"2026-09-17T02:31:43.962353158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0aba0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-ms9kkl5qh4rm\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\",\n      \"name\": \"7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:38.567378599Z\",\n      \"endTime\": \"2026-09-17T02:31:43.962353158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cdc740\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9lhfrbsqkhjk\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\",\n      \"name\": \"39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:27.808727421Z\",\n      \"endTime\": \"2026-09-17T02:31:33.989605445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cdc740\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9lhfrbsqkhjk\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\",\n      \"name\": \"39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:27.808727421Z\",\n      \"endTime\": \"2026-09-17T02:31:33.989605445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"02219053511d8a21950017349ee31494aa267bbe162808b309c406afe57bd88e","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"0ea6e92c609e741767496ed0bca205dd97df87334b05401151ca3d9e4297e4cc","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"139bcc210692fb4c4608f88c1e62ffbae5cc26a9ea38e8a550f86dc64be5bb37","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"2161ec92ec6ee1b1f4126d8043c75c164aa59ccae1d75ae8fdecc279478acefb","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"24e450fae1afaf3835ed114031825dba965e2638d9e2a4b78361d363bc7d139c","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"284fd36dc5ef5d74feffb6a93226bcfff835efdad8f1f85bdf032a8053cc1b6a","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"2e37c10dee44a98ad4c5d8e21f945e822bc6d1d157c0aec08182d653353bf301","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"2f9108fe18776541cbc85ef378b8f266052150ee76b5c630519c31e933ddb5b9","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"39e9738b3570d1ad49126bbe45de1c14c1222d36b148883b549374de80d01c71","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"417a674aaec0692852dbca6d977cc89fe23494c77a5a0606b6bafecdf7722e62","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"451e15ada92473b46c37ac7d85751c6c542d9156788736edb08510846c807aa5","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"593e3571cb87eb1c990363d1e184cfd091bfa920f1d8233ab9b8bffafc8753f4","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"626800ce28af5178db2305452aa40119998bcbdd63d53a7168853b1c7e730bd3","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"671e3e4773a45e7910b33f7d44f7a3f0d6be8188410cc6c10ced0d14da59716b","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"6d8549ce7ce3d738e38f7dd73b06fae91adfeb0e1534bba97833a20ae84d2f8a","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"7abcaf7cf828a86d797ada0bafb042a8494db58e91375d1805c53de72f0578cb","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"91e2917906e893c4adf51d8e3ad38930cff19ee5cb3009d04823d9b45fe41908","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"9e699244c91afd46d8472b6ccd3aa4e2a0f99db87af440b007dcf8abcbb3ce6d","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"9e87469877ef0917409c14d4d33b60f1b2ccdcdd81485077bc197c0d9d5a5e97","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"a8a2c2adbed4b3b26d51343db3e33c8589838c2cbac6661fcee12fbe4bf879fe","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"aa2abdeccf981853abdf8d84e7171612f5443e4af2a9fb66bb42c41ddd759ab7","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"c8695c56c9968b93a6495331c031bfc4f9defddf58bdeb3d72666b657cc7d8c5","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"d166da666940169c48a559831401ddc71f07bc674b8bac26199e6f32e66bd324","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"db18d58c7751ab0b22cdad498cec1c389a7f4bcbaed367974d93a33204cc2434","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"e19ccc38c497a63fa54dc54b1393fcc47c00e94ab08b2c73966853333434a251","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"f188a0c69f93a586d8064b19fecfad91c62cb72340482fa8197c45527b731e94","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"1081dc1f7ba277c79eab9706dd413878fd827376a9850ff6a2e23d1b663c5155","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"51fb4ad7b246c62ce6ea07724802f78706075364625cc9b0dc59f38a98c159cf","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"52a3e55e5accf233dc0c4dcc31ad7c4bd1abab3ca6c7016627f681951931597f","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"8eb85a382d8ce34de42657ea34ddc180803991e921b8e70810a099fe550ece42","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"b323b711e4f15f03edb6542a780e728151255e7713e3d463bc33d5461bb8ccbf","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"daf2b2ff378776852d48a0372497ce4c540ea79fca95c306e900dac48324c4cd","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"ea6f45f7f8317d27652ca2ba1a9ea637325a720fb6e451e052632539521bf95b","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":22,"runs_affected":20,"failed_runs":20,"run_impact_percent":2.7472527472527473,"anchor_occurrences":22,"after_last_push_count":12,"also_in":["INT","PROD","STG"],"trend_counts":[0,0,0,6,6,4,1,5],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":20,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00100fa80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-xdxwcdw5rptk\" to finish creating: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\",\n      \"name\": \"f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T00:27:13.938506094Z\",\n      \"endTime\": \"2026-09-21T00:46:21.511326736Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00100fa80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-xdxwcdw5rptk\" to finish creating: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\",\n      \"name\": \"f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T00:27:13.938506094Z\",\n      \"endTime\": \"2026-09-21T00:46:21.511326736Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101c6e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-hxgb2rxvk82d\" to finish creating: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ed88cf8-d1af-4b17-9bae-6034a71c6604\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ed88cf8-d1af-4b17-9bae-6034a71c6604\",\n      \"name\": \"2ed88cf8-d1af-4b17-9bae-6034a71c6604\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-20T20:48:17.088169395Z\",\n      \"endTime\": \"2026-09-20T21:07:20.582868448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101c6e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-hxgb2rxvk82d\" to finish creating: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ed88cf8-d1af-4b17-9bae-6034a71c6604\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ed88cf8-d1af-4b17-9bae-6034a71c6604\",\n      \"name\": \"2ed88cf8-d1af-4b17-9bae-6034a71c6604\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-20T20:48:17.088169395Z\",\n      \"endTime\": \"2026-09-20T21:07:20.582868448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-ggj24g\" for cluster \"cluster-candidate-4-20-ggj24g\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000219dc0\u003e: \n    failed to create NodePool np-ggj24g: failed waiting for nodepool=\"np-ggj24g\" for cluster \"cluster-candidate-4-20-ggj24g\" in resourcegroup=\"rg-candidate-4-20-686sncjvb2sx\" to finish creating: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60924801-61a0-4bc9-912c-bcdf013ae1f4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60924801-61a0-4bc9-912c-bcdf013ae1f4\",\n      \"name\": \"60924801-61a0-4bc9-912c-bcdf013ae1f4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-20T20:06:27.477664357Z\",\n      \"endTime\": \"2026-09-20T20:25:30.377289875Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-ggj24g\" for cluster \"cluster-candidate-4-20-ggj24g\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000219dc0\u003e: \n    failed to create NodePool np-ggj24g: failed waiting for nodepool=\"np-ggj24g\" for cluster \"cluster-candidate-4-20-ggj24g\" in resourcegroup=\"rg-candidate-4-20-686sncjvb2sx\" to finish creating: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60924801-61a0-4bc9-912c-bcdf013ae1f4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60924801-61a0-4bc9-912c-bcdf013ae1f4\",\n      \"name\": \"60924801-61a0-4bc9-912c-bcdf013ae1f4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-20T20:06:27.477664357Z\",\n      \"endTime\": \"2026-09-20T20:25:30.377289875Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"297f5e4b1d1f4952837a1cae50b737b42a9bc431965f0ebd876867d9c038a0d0","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","signature_id":"fec305f359038a8b537a558ba87b3d6d7b2c427d51eb493f15dad5445be91e77","pr_number":7030,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"08debcb95fe76787cc5e7470c732a78966ba147069573bb898c775a88f1cff3e","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"531647f22030c4eb57aa8d0ce0f1192ad57dde45a170d15dbfbc6d84af797341","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"e54b973f5d0dfed8acc1e8e3583260026a6a968d1ae6ee942caf592be98feaeb","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7021/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101282005600702464","occurred_at":"2026-09-19T12:07:24Z","signature_id":"223d82754cf5f8fcd7f066ac38ba1054fd1ce9d60af827bb046102fd2bc80482","pr_number":7021,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101033546570272768","occurred_at":"2026-09-18T19:39:52Z","signature_id":"50e9ecb754e15646a314b5b99f973f998a45c92fb4fac3293bbdba2c5f6a2e7e","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","signature_id":"79534399cf979be34c0ffb48ddc762d6841c4a16f58dc222924b468f595a7adb","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6962/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100817340928102400","occurred_at":"2026-09-18T05:20:59Z","signature_id":"fd810f6089fd17b9ae89f5ca382f4365705ce74f6259d695bca5e674d7783e53","pr_number":6962,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6813/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815351901065216","occurred_at":"2026-09-18T05:13:05Z","signature_id":"1b937eb2ed1eae4fc31ae0598f8de6ce2076a500845fc3ea7dcd762870526b2e","pr_number":6813,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100691156575719424","occurred_at":"2026-09-17T20:59:35Z","signature_id":"13adef91a90ab725dab440fb0cd3d153b3bf6f23ccf1898e5f571b9c23787d80","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100678231362375680","occurred_at":"2026-09-17T20:03:18Z","signature_id":"14578415f39945108c1077bc9568d80eb9c57e111ae4ccc29677f2723df82376","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6985/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100532310913847296","occurred_at":"2026-09-17T10:28:23Z","signature_id":"11862ebb0b64914c35ffbbc35bccd2d2fc728333e2922d73613bd21bacfcabf1","pr_number":6985,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100523779246526464","occurred_at":"2026-09-17T09:54:29Z","signature_id":"7fc0697aa9693154afc06e12851d0da5c2c6857a0d678a54183094477e417af7","pr_number":6879,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494069930135552","occurred_at":"2026-09-17T07:56:26Z","signature_id":"41623d8231704f6b3918018ef069742c2b6b4139b736f8af0a971c826a53f099","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100469063225446400","occurred_at":"2026-09-17T06:17:04Z","signature_id":"41037677a8ddb37389a2d414387cf88a833a474c6bb9aecf6b5fe7288cd9a6b8","pr_number":6986,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","signature_id":"11b5fac45f89b3b06fa90df1c77a750f6663c0d038abc00b0bfa209370a40fa8","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100212360202424320","occurred_at":"2026-09-16T13:16:57Z","signature_id":"fca0d524c5ac315fe836fb88fd7ec89c3926bdf9a49356a4f7143c3576597659","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100177539514241024","occurred_at":"2026-09-16T10:58:39Z","signature_id":"4035d63aa7b2b3c2922026477a9808247a9246b7c7d1f67b92c709fe54aaae23","pr_number":6952,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100170313131102208","occurred_at":"2026-09-16T10:29:56Z","signature_id":"f02c430f8d6e71033486f073da10a12fce79eb5455b645129c51f20c2adc1b42","pr_number":6886,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100164075236364288","occurred_at":"2026-09-16T10:05:09Z","signature_id":"05b8123730c99aae90bf4a2eb92268ce4dedb8de1e1191eb555f549c617a26dd","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"b85f0369f91fe32b9c9c2062c2413bb6bf553dd1bd69f6e51a7a27c7ba432aee","pr_number":6791,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":20,"runs_affected":20,"failed_runs":20,"run_impact_percent":2.7472527472527473,"anchor_occurrences":20,"after_last_push_count":9,"trend_counts":[0,0,0,6,6,3,2,3],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":20,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":20}],"full_error_samples":["alert fired 24 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T01:01:07Z\n  Ended:       2026-09-21T01:19:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T01:01:07Z\n  Ended:       2026-09-21T01:19:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-21T01:02:09Z\n  Ended:       2026-09-21T01:16:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-21T01:02:09Z\n  Ended:       2026-09-21T01:17:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-21T01:02:09Z\n  Ended:       2026-09-21T01:16:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-21T01:02:09Z\n  Ended:       2026-09-21T01:17:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-21T01:03:05Z\n  Ended:       2026-09-21T01:19:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-21T01:04:08Z\n  Ended:       2026-09-21T01:16:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-21T01:04:08Z\n  Ended:       2026-09-21T01:16:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-21T01:04:08Z\n  Ended:       2026-09-21T01:16:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-21T01:04:08Z\n  Ended:       2026-09-21T01:19:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-21T01:04:08Z\n  Ended:       2026-09-21T01:19:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-09-21T01:04:08Z\n  Ended:       2026-09-21T01:19:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-09-21T01:04:08Z\n  Ended:       2026-09-21T01:16:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-09-21T01:04:08Z\n  Ended:       2026-09-21T01:19:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-09-21T01:04:08Z\n  Ended:       2026-09-21T01:17:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-09-21T01:04:08Z\n  Ended:       2026-09-21T01:17:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-09-21T01:04:08Z\n  Ended:       2026-09-21T01:19:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-09-21T01:04:08Z\n  Ended:       2026-09-21T01:17:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-09-21T01:05:07Z\n  Ended:       2026-09-21T01:19:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 21:\n  State:       Resolved\n  Started:     2026-09-21T01:05:07Z\n  Ended:       2026-09-21T01:17:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 22:\n  State:       Resolved\n  Started:     2026-09-21T01:05:07Z\n  Ended:       2026-09-21T01:19:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 23:\n  State:       Resolved\n  Started:     2026-09-21T01:06:08Z\n  Ended:       2026-09-21T01:19:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 24:\n  State:       Resolved\n  Started:     2026-09-21T01:06:08Z\n  Ended:       2026-09-21T01:19:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T00:49:52Z\n  Ended:       2026-09-21T01:04:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4251392/providers/microsoft.monitor/accounts/hcps-j4251392\", namespace=\"ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-5tv64\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-20T15:26:04Z\n  Ended:       2026-09-20T15:41:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6358528-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.107:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6358528/providers/microsoft.monitor/accounts/hcps-j6358528\", namespace=\"ocm-arohcpci01-2suq8o78o0jbc33n4l57r98vk1s6tm51-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-x5q5k\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2suq8o78o0jbc33n4l57r98vk1s6tm51-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"a34c4272b290720aafd3b29a745fe60b6976f6c8d6360946f5a75edf1096c9ec","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"a2c1fbf6e64cddf7a46eaafcc044b53f14f98a5940a39f4dbc58486310e950ce","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101673315566358528","occurred_at":"2026-09-20T14:02:20Z","signature_id":"84036f94843f01e49f13138f55d77ae74e49b96060ac2cc402fa01f03e462128","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7021/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101139510631862272","occurred_at":"2026-09-19T02:41:11Z","signature_id":"c3b64b82bf476afdd548a6ba802c6e747707cdb32e962d3503a8a71c8445d0d7","pr_number":7021,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101105796258467840","occurred_at":"2026-09-19T00:27:13Z","signature_id":"abf9afbaec1b5ab998a4041de0af4ad87a83bfb4310e5d5ad18de08964bcd3f7","pr_number":7019,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","signature_id":"61a9d86a254d9ba501b68a3f3cb4e321b30d8b314bb25ec5b72e60af5be20fd0","pr_number":7012,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"035123031fa1d20ace889ddb3bd473206d7c64b2e0e69a0775e5e52e6c40e898","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"b30257b5749c88e7acf4c108618d88d0f18563968e68f249086f8fef88556ee6","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685654236073984","occurred_at":"2026-09-17T20:37:43Z","signature_id":"a25eab3d2aea0534a58060ed60f2bfb99bd3c649f3ff0b27d214e0eebb1f8f68","pr_number":7000,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100523023395196928","occurred_at":"2026-09-17T09:51:23Z","signature_id":"98cf6a34d877442691cd61bb1e77f0e720f13f9f1c1c8cdddc6ad3341afc78ff","pr_number":5724,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"a85ef3a241aa9fee6ef7ff6043d1020492ba3c7fed3de7483ade938f342c7c3e","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100478808179085312","occurred_at":"2026-09-17T06:55:45Z","signature_id":"a995d8a36ccb5cf52cd9d35f8ae8324aa7b3c5eb47ebb28e229463bfb8fb04c6","pr_number":6988,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"6133b883dec1a7ae7199f6897e93bfe8938d4615907efd1f6e96f8f531de45b7","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100385816462233600","occurred_at":"2026-09-17T00:46:16Z","signature_id":"54b42f0c1ae5cc077af2dfa797005520003ed2bd8ca931aabcdf79c00d97c263","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"0d965c3262ee69273c199442b4bed217de7dd1b47c2e6f6c022895186517f6a2","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"314e25f4f6bda2d5748f9fe5aa89f88eaeb725849f7741eef5e0355c2394136c","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"d5ed3091019088eb341f9671128472e74176d896eca057722a5636069c20079f","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","signature_id":"18b82d4d472bbc8192b3a0be8a58b9abe4cf0da273e711ee46164983b7c2c490","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"278a7d810c61d983530553cb6ff8366e5a4d79b77f9c15651d9ebcb3dbeb9e4e","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"89db9db5751f9f51402e00b217747d3b3ea1c62487d699b8eeaa41599ef274e2","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":20,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":20,"after_last_push_count":20,"trend_counts":[0,0,0,0,0,0,0,20],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:76]: failed to create customer resources for private KAS cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f02d60\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-private-kas-m75gws\" in resourcegroup=\"private-kas-s98lp67jv589\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-kas-s98lp67jv589/providers/Microsoft.Resources/deployments/customer-infra-private-kas-m75gws/operationStatuses/08584116992457285882\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"InternalServerError\\\",\\r\\n        \\\"message\\\": \\\"An error occurred.\\\",\\r\\n        \\\"details\\\": []\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:76]: failed to create customer resources for private KAS cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f02d60\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-private-kas-m75gws\" in resourcegroup=\"private-kas-s98lp67jv589\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-kas-s98lp67jv589/providers/Microsoft.Resources/deployments/customer-infra-private-kas-m75gws/operationStatuses/08584116992457285882\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"InternalServerError\\\",\\r\\n        \\\"message\\\": \\\"An error occurred.\\\",\\r\\n        \\\"details\\\": []\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a11f20\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-private-ingress-ltthln\" in resourcegroup=\"private-ingress-6mv7ddflq9wr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-6mv7ddflq9wr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-ltthln/operationStatuses/08584116992436980878\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"InternalServerError\\\",\\r\\n        \\\"message\\\": \\\"An error occurred.\\\",\\r\\n        \\\"details\\\": []\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a11f20\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-private-ingress-ltthln\" in resourcegroup=\"private-ingress-6mv7ddflq9wr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-6mv7ddflq9wr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-ltthln/operationStatuses/08584116992436980878\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"InternalServerError\\\",\\r\\n        \\\"message\\\": \\\"An error occurred.\\\",\\r\\n        \\\"details\\\": []\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:110]: failed to create customer resources for cluster-subnet-reuse\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c12720\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-subnet-reuse-h6k9tb\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xf4hx8gcmq7v\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-xf4hx8gcmq7v/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-h6k9tb/operationStatuses/08584116992459228470\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"InternalServerError\\\",\\r\\n        \\\"message\\\": \\\"An error occurred.\\\",\\r\\n        \\\"details\\\": []\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:110]: failed to create customer resources for cluster-subnet-reuse\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c12720\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-subnet-reuse-h6k9tb\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xf4hx8gcmq7v\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-xf4hx8gcmq7v/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-h6k9tb/operationStatuses/08584116992459228470\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"InternalServerError\\\",\\r\\n        \\\"message\\\": \\\"An error occurred.\\\",\\r\\n        \\\"details\\\": []\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"1a79c558038cb73ff72359e020fc0d6f9f740e31cbd0af1daebaf91ad6c65fa3","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"1b003937044cfa4fcd5a2675364491c0f6ffb7db83c6398081a44e0f34d6bb6b","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"268106fe2eb57086c4eef48987e800efaac560af7778223cb42b3520a44d6749","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"2b974162c7e6d97b0de4aea836205f4f789664585153978c18ab76f65fd6ff59","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"42440a2d7bb764d5edcdcdcd65e32a762540470eaa9f7ce1559f0c2a375c81a0","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"454410db4b0c9c7e89e9e307d8cabd3be3a5cc6c620a919683872059242695a3","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"56478c58658652b4bc1945a590ac05ad32b42a6f22c2e138039cb7ee3c4cbfac","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"5d0ca65ae867bd15279982b3a25e60bccfc658f662dfee162bb114f2ddaccb2c","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"73c41b364dee2fd3eb27b4ccaa57fb601deed774cfcc5111654a86e31aa1c0ef","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"75b9233e5419e103c738cc032e9a0d4380e13a4936bd5bad2e51413dae0bf26f","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"81b579506962c7694460b4e0ba3269306f93397ebe18eb85545cae4066854205","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"91a719e5ea3aea8b9b0e188fedf3ccfcff48a4a62dc5fc1afe26e222804d6227","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"9db4a31a545e08b96a9f9b2a51e0ec6fa079b7393b1270544cac4af6f6246fd5","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"a186884d8b4f85878973e1e8bb26766fcd36f0f99234916982d31c2386197c2a","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"a21cc835fa8cdb2ed9033f1ca18d1eb7db85fc60669d3f2f049f9f0f8e99c24d","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"ab6420d788f7161106d32d50e735d0b55e0a7d0b7c8c03f40db9586c7ac5332f","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"b9513cbbafa37244dbcd3207b8b1836ea824ef6ed8a8fa6ee5005da287c245c1","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"bd221a69442a6d79a216e48307db3abafc2724516fd0d3101335d3d4bb683a2d","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"c3ed73d691c5929cdf2946a476c8b001c9536e6031f54f65e36b74ff7e76c90f","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"d798f3f3a8a081a020bf7521957c622331f9901d07425be35f0617500d755d67","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":15,"runs_affected":15,"failed_runs":15,"run_impact_percent":2.0604395604395602,"anchor_occurrences":15,"after_last_push_count":11,"trend_counts":[0,0,0,5,1,4,1,4],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":15,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":15}],"full_error_samples":["alert fired 24 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T00:46:08Z\n  Ended:       2026-09-21T01:14:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T00:47:08Z\n  Ended:       2026-09-21T01:18:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-21T00:47:09Z\n  Ended:       2026-09-21T01:15:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-21T00:47:09Z\n  Ended:       2026-09-21T01:15:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-21T00:47:09Z\n  Ended:       2026-09-21T01:18:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-21T00:48:11Z\n  Ended:       2026-09-21T01:18:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-21T00:48:11Z\n  Ended:       2026-09-21T01:15:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-21T00:48:11Z\n  Ended:       2026-09-21T01:15:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-21T00:49:07Z\n  Ended:       2026-09-21T01:15:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-21T00:49:07Z\n  Ended:       2026-09-21T01:18:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-21T00:49:07Z\n  Ended:       2026-09-21T01:15:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-21T00:49:07Z\n  Ended:       2026-09-21T01:14:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-09-21T00:49:07Z\n  Ended:       2026-09-21T01:15:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-09-21T00:50:11Z\n  Ended:       2026-09-21T01:18:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-09-21T00:50:11Z\n  Ended:       2026-09-21T01:16:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-09-21T00:50:11Z\n  Ended:       2026-09-21T01:19:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-09-21T00:50:11Z\n  Ended:       2026-09-21T01:15:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-09-21T00:50:11Z\n  Ended:       2026-09-21T01:14:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-09-21T00:50:11Z\n  Ended:       2026-09-21T01:14:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-09-21T00:50:11Z\n  Ended:       2026-09-21T01:14:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 21:\n  State:       Resolved\n  Started:     2026-09-21T00:51:12Z\n  Ended:       2026-09-21T01:18:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 22:\n  State:       Resolved\n  Started:     2026-09-21T00:51:13Z\n  Ended:       2026-09-21T01:17:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 23:\n  State:       Resolved\n  Started:     2026-09-21T00:51:13Z\n  Ended:       2026-09-21T01:14:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 24:\n  State:       Resolved\n  Started:     2026-09-21T00:55:14Z\n  Ended:       2026-09-21T01:14:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T00:35:55Z\n  Ended:       2026-09-21T01:02:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4251392/providers/microsoft.monitor/accounts/hcps-j4251392\", namespace=\"ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-5tv64\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T00:35:55Z\n  Ended:       2026-09-21T01:06:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4251392/providers/microsoft.monitor/accounts/hcps-j4251392\", namespace=\"ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-5tv64\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 20 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-20T18:09:57Z\n  Ended:       2026-09-20T18:28:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-20T18:09:57Z\n  Ended:       2026-09-20T18:28:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-20T18:09:57Z\n  Ended:       2026-09-20T18:26:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-20T18:09:57Z\n  Ended:       2026-09-20T18:28:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-20T18:09:57Z\n  Ended:       2026-09-20T18:26:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-20T18:09:57Z\n  Ended:       2026-09-20T18:26:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-20T18:10:51Z\n  Ended:       2026-09-20T18:28:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-20T18:10:51Z\n  Ended:       2026-09-20T18:28:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-20T18:10:51Z\n  Ended:       2026-09-20T18:26:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-20T18:10:52Z\n  Ended:       2026-09-20T18:26:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-20T18:11:50Z\n  Ended:       2026-09-20T18:28:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-20T18:11:50Z\n  Ended:       2026-09-20T18:28:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-09-20T18:11:50Z\n  Ended:       2026-09-20T18:28:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-09-20T18:11:50Z\n  Ended:       2026-09-20T18:28:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-09-20T18:11:50Z\n  Ended:       2026-09-20T18:26:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-09-20T18:11:50Z\n  Ended:       2026-09-20T18:28:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-09-20T18:12:50Z\n  Ended:       2026-09-20T18:26:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-09-20T18:12:50Z\n  Ended:       2026-09-20T18:26:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-09-20T18:12:50Z\n  Ended:       2026-09-20T18:26:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-09-20T18:12:50Z\n  Ended:       2026-09-20T18:26:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"59cd894a17dcad23b6224f622452e17a2a68039a078585dd58b1284a023c6bf0","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"386fa5f6fc68894d00954640db0f0852279a2e3a87df5f417fecec60fd94a6bb","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"8da3d9c87ac6e15f84a876a8ec0eb6d135540c742388bef66277cd5947ec5e3b","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101673315566358528","occurred_at":"2026-09-20T14:02:20Z","signature_id":"e66e5d2206cbbff91631193f79b282c5f04f3b6efea313eb1c9fc43af3b6f8ad","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101105796258467840","occurred_at":"2026-09-19T00:27:13Z","signature_id":"52f5e8f7abd2c0f726dc6b21e46d6b4ff7d5890afeac559b9680bd0bad80a332","pr_number":7019,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","signature_id":"7dead5f17d20c600ef9a15a67c9915db561b768a43db7c68ada9d07b7b90dfc6","pr_number":7012,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"77db7532989dc335d44e3fd0e347138b7a4aa37045fd816ecfc3761a0789b616","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100877868748771328","occurred_at":"2026-09-18T09:21:30Z","signature_id":"077ee51cbdfb7388e2c8c28578e14f7c26aad49cf417b4ab9a95eda6e094ce1d","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","signature_id":"32caccfe24784ea73efa24f662aed9cbdd2ff753b7f3457b2fae68c38d3d3c3d","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100478808179085312","occurred_at":"2026-09-17T06:55:45Z","signature_id":"78b86d170894f1306383271eb64e5c7231bc9db1d4dade7d27fb2bd571aeb710","pr_number":6988,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","signature_id":"f812f1c6853ffe5ec0bbae33597f7ac343264a944ebc861a726251e8dce290ce","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"b995781a6685881ffdb2f7b53822eec544ca3973eb89f51c67b3781f51c35c18","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","signature_id":"67413ecc4fbc527d42baea24debbddc611e8d90812ae968414dd499232adf3e3","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6726/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100185438990897152","occurred_at":"2026-09-16T11:30:02Z","signature_id":"e361abead0603e0446eac1b70560a55fbc49aa8e7044098f3920029a67178a2a","pr_number":6726,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132115432607744","occurred_at":"2026-09-16T07:58:09Z","signature_id":"2202a1a6bca53c40fa9b99a6151f7edd9b592782635ef9dfab434b81cf067278","pr_number":6964,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":15,"runs_affected":15,"failed_runs":15,"run_impact_percent":2.0604395604395602,"anchor_occurrences":15,"after_last_push_count":6,"trend_counts":[0,0,0,10,5,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":15,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":6},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":9}],"full_error_samples":["time=2026-09-17T22:08:23.042Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T22:08:24.010Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-17T22:08:24.819Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=949ef12d6ab429b901225672cf02dc8c2e8f5344e8e4f364f694e8721bc1ed4e portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0316160-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F949ef12d6ab429b901225672cf02dc8c2e8f5344e8e4f364f694e8721bc1ed4e\ntime=2026-09-17T22:38:22.805Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-09-17T21:21:29.249Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T21:21:29.969Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-17T21:21:30.723Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=63b12b492d5d28b0b0b18408fa469c00c31993867db99322c5926f55f6db447a portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6654848-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F63b12b492d5d28b0b0b18408fa469c00c31993867db99322c5926f55f6db447a\ntime=2026-09-17T21:51:28.943Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-09-17T21:19:38.472Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T21:19:39.145Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-17T21:19:39.871Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=f518dca3cc0829098231fe8a127f84b76b966486264714a9e41a0d1fd09997db portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0025600-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff518dca3cc0829098231fe8a127f84b76b966486264714a9e41a0d1fd09997db\ntime=2026-09-17T21:49:38.148Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100701574740316160","occurred_at":"2026-09-17T21:40:59Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100691038216654848","occurred_at":"2026-09-17T20:59:07Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100689397040025600","occurred_at":"2026-09-17T20:52:35Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495912190087168","occurred_at":"2026-09-17T08:03:45Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488996214804480","occurred_at":"2026-09-17T07:36:16Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100332359231475712","occurred_at":"2026-09-16T21:13:51Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100270038647836672","occurred_at":"2026-09-16T17:06:12Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100228886901035008","occurred_at":"2026-09-16T14:22:39Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6779/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100198040831791104","occurred_at":"2026-09-16T12:20:07Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6779,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194885893099520","occurred_at":"2026-09-16T12:07:35Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194111955931136","occurred_at":"2026-09-16T12:04:19Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100192526756483072","occurred_at":"2026-09-16T11:58:12Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100190163022909440","occurred_at":"2026-09-16T11:48:49Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6694/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100169554075324416","occurred_at":"2026-09-16T10:26:55Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6694,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100119215724826624","occurred_at":"2026-09-16T07:06:53Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6972,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":13,"runs_affected":13,"failed_runs":13,"run_impact_percent":1.7857142857142858,"anchor_occurrences":13,"after_last_push_count":7,"trend_counts":[0,0,0,4,2,1,3,3],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":13,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":13}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-21T01:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4251392-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-5hkmr\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-5hkmr (namespace arobit) on cluster ci01-j4251392-svc has averaged 126.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-20T18:55:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j3380352-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-qgqnm\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-qgqnm (namespace arobit) on cluster ci01-j3380352-svc has averaged 126.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-20T16:39:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9006208-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-vbl5w\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-vbl5w (namespace arobit) on cluster ci01-j9006208-svc has averaged 126.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"39e21beab76cee821e0ed80a6601d86d04d54351b87f382522d67356ee5fa584","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"b4c5714662b6eebf9aeab7aa2383a89d5f76557fa6ae1c0f2264a9d31c883f89","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101683713959006208","occurred_at":"2026-09-20T14:43:39Z","signature_id":"fb812e0e230afdfaf46a5827aae975ba58d618c82618d4f8ed79f5468297e6a0","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101401417230061568","occurred_at":"2026-09-19T20:01:54Z","signature_id":"483f9d46f66e016c010baab6f6ef4cd3c30895548f7661edb8af252882a5defa","pr_number":7019,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101360030631596032","occurred_at":"2026-09-19T17:17:27Z","signature_id":"4293148a17db814de2b7ff5cdc6d2742b3ee0185946474acd473de51fd8fa90d","pr_number":7019,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101255238609539072","occurred_at":"2026-09-19T10:21:02Z","signature_id":"3e718e1923193d2fd6442f38c76702bb255b7621b3d4f0d6989a1706f04ef7ce","pr_number":6925,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","signature_id":"476aed8123ee528db23f3663c1332194314ed537bec78310ea649964cd6c166f","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721091256258560","occurred_at":"2026-09-17T22:58:32Z","signature_id":"fffbb7c99a9e3fbfbbfb27296f90fe1eaf381e133c47c2097dc7c5b3440df9c4","pr_number":7000,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","signature_id":"f1e05c2e51f466f034476da1ae61bfe77799ca5b43dd98ad6f2836fda4da614d","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"7cedda9870b8f7ce3e9e32ac805564cf0bac96565ca3328c26875a1d70efde15","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309676229922816","occurred_at":"2026-09-16T19:43:43Z","signature_id":"3a277ae7c258e0424f569baffe0a35ab11aec77aa3384cc607f2298340f9545f","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100285588732121088","occurred_at":"2026-09-16T18:08:00Z","signature_id":"040c54783eeb571a32188be480f0e0617f1911469f8d08b133b147d373df8efb","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100273649465430016","occurred_at":"2026-09-16T17:20:33Z","signature_id":"de52e1f3238a7b10dc025230a0bc30a7b49fe207de3da3e23620f54404272671","pr_number":6894,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":11,"runs_affected":11,"failed_runs":11,"run_impact_percent":1.510989010989011,"anchor_occurrences":11,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,3,3,5],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":11,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":11}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-20T21:15:25Z\n  Ended:       2026-09-20T21:27:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j1718656-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n  Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j1718656-svc).","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-20T18:17:23Z\n  Ended:       2026-09-20T18:33:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j6867712-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n  Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j6867712-svc).\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-20T18:17:23Z\n  Ended:       2026-09-20T18:32:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j6867712-svc\", component=\"frontend\", method=\"delete\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j6867712-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-20T13:27:17Z\n  Ended:       2026-09-20T13:41:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j2758528-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j2758528-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7035/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101762182411718656","occurred_at":"2026-09-20T19:55:27Z","signature_id":"993e0edbc2802d27b58b7315a4badc5470fe6efc5c6616983981032abe8b49db","pr_number":7035,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101715422586867712","occurred_at":"2026-09-20T16:49:39Z","signature_id":"a1c099010ae016739013a1e87affb32949a8f1160e4fb45e709b3366b93a0cff","pr_number":7033,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643653972758528","occurred_at":"2026-09-20T12:04:28Z","signature_id":"f483622241db14070571766c5ea34d86fd6f53d96057f1964f32d162917e2451","pr_number":7030,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101634420027953152","occurred_at":"2026-09-20T11:27:46Z","signature_id":"5b8bff552d46579784f79a0d3b3120eddbccf7ef351d544b8715f930813682e3","pr_number":7026,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","occurred_at":"2026-09-20T09:53:04Z","signature_id":"f6030bcc4e4cc5c82d977d09b32388275720da97127703ae213df38d33273f13","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101441967811465216","occurred_at":"2026-09-19T22:43:02Z","signature_id":"1a0e62e2598794fc1e8ca8c1c4c96323a3ea21d8d2d742822823aa3aa712e5ed","pr_number":7019,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7021/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101397927778848768","occurred_at":"2026-09-19T19:48:02Z","signature_id":"2262ed71d953617413345641743587826cf9cc96520d429984a293e722318bd3","pr_number":7021,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101255238609539072","occurred_at":"2026-09-19T10:21:02Z","signature_id":"7fbc94e6b577608063b8ffe14b6e44d6f704c3d3b69ea1dbaae3dbea757e4c62","pr_number":6925,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7018/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101049380864790528","occurred_at":"2026-09-18T20:43:02Z","signature_id":"900336073fa1807e3fa5eb270059d6e459df8256995e512e7d9ae978fff82f8c","pr_number":7018,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","signature_id":"f6e1d34656f1ce37365020fae4967ac3b4ae64788fb876cb79e6b050b731b2fe","pr_number":7010,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100917027756576768","occurred_at":"2026-09-18T11:57:07Z","signature_id":"85fef6c172f8238e32065ce9be94996092fd65499dbb48afe109749b4511663e","pr_number":6939,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":10,"runs_affected":10,"failed_runs":10,"run_impact_percent":1.3736263736263736,"anchor_occurrences":10,"after_last_push_count":4,"trend_counts":[0,0,0,3,1,4,1,1],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":10,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":10}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-20T17:13:02Z\n  Ended:       2026-09-20T17:29:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5465216-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-19T11:30:53Z\n  Ended:       2026-09-19T11:44:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4648192-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T09:27:59Z\n  Ended:       2026-09-18T09:34:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6818048-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101705684675465216","occurred_at":"2026-09-20T16:10:57Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":7030,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7013/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101255401264648192","occurred_at":"2026-09-19T10:21:41Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":7013,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100862764498030592","occurred_at":"2026-09-18T08:21:29Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":6984,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712898639695872","occurred_at":"2026-09-17T22:25:59Z","signature_id":"3b128919de15d094b3050816e84bfdc2f25e06d586dd063b36f0c63378fdbfef","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"660cafcaf8e83347ebad23ce568c8f5b0468c2f68c8566340ecac05676075f1d","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"fe76c2aef36843354f881968133b4c2405287f6a2f2ee3e1e7ab8ab4682897ce","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100153419804184576","occurred_at":"2026-09-16T09:22:48Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":10,"runs_affected":10,"failed_runs":10,"run_impact_percent":1.3736263736263736,"anchor_occurrences":10,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[0,0,0,4,3,2,1,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":10,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":10}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7021/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101139510631862272","occurred_at":"2026-09-19T02:41:11Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":7021,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","signature_id":"e3628da3891e1ec49619dde7547596c92e9e1c1cfe73fca8833c1206090506a8","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488796448493568","occurred_at":"2026-09-17T07:35:28Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"b9b50758a1aa006036c7c1ee58af90cd509c7b1900722ef0229b46ed2f5e8563","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"b9b50758a1aa006036c7c1ee58af90cd509c7b1900722ef0229b46ed2f5e8563","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100196783199096832","occurred_at":"2026-09-16T12:15:07Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"817dbbd2f4e987cf783899d57ba9ab565857a252a91d7741c25fbfc506aff5e8","pr_number":6879,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":10,"runs_affected":9,"failed_runs":9,"run_impact_percent":1.2362637362637363,"anchor_occurrences":10,"after_last_push_count":4,"trend_counts":[0,0,0,5,3,2,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":9,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:111]: failed to create node pool \"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001015700\u003e: \n    failed to create NodePool np-zrxbbn: failed waiting for nodepool=\"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\" in resourcegroup=\"rg-hs-pre--gs2tq8nbktl2\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n      \"name\": \"4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T03:55:06.959884593Z\",\n      \"endTime\": \"2026-09-18T03:55:51.664849739Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:111]: failed to create node pool \"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001015700\u003e: \n    failed to create NodePool np-zrxbbn: failed waiting for nodepool=\"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\" in resourcegroup=\"rg-hs-pre--gs2tq8nbktl2\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n      \"name\": \"4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T03:55:06.959884593Z\",\n      \"endTime\": \"2026-09-18T03:55:51.664849739Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017ae060\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-r4g2f9rh44ss\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n      \"name\": \"499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T03:54:50.355767916Z\",\n      \"endTime\": \"2026-09-18T03:55:21.65277157Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017ae060\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-r4g2f9rh44ss\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n      \"name\": \"499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T03:54:50.355767916Z\",\n      \"endTime\": \"2026-09-18T03:55:21.65277157Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001d94018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-g7cwbl5k67j4\" to finish creating: GET https://rp.j8532992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbeee8b5-27cd-473e-a9c2-df2114c56f2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbeee8b5-27cd-473e-a9c2-df2114c56f2b\",\n      \"name\": \"fbeee8b5-27cd-473e-a9c2-df2114c56f2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T00:05:39.3471997Z\",\n      \"endTime\": \"2026-09-18T00:06:19.6272248Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001d94018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-g7cwbl5k67j4\" to finish creating: GET https://rp.j8532992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbeee8b5-27cd-473e-a9c2-df2114c56f2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbeee8b5-27cd-473e-a9c2-df2114c56f2b\",\n      \"name\": \"fbeee8b5-27cd-473e-a9c2-df2114c56f2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T00:05:39.3471997Z\",\n      \"endTime\": \"2026-09-18T00:06:19.6272248Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","signature_id":"6acf378b1d751cabdd2cb29f20e478c64231e5b033214257ad51d0d59a0c2ba7","pr_number":6972,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","signature_id":"f081e78216b7bd4a54e652f3dfd3a2cf8cf19638bb76bda01151759533a92078","pr_number":6972,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721125368532992","occurred_at":"2026-09-17T22:58:38Z","signature_id":"3ad0c7f8513c2d305c0610677386a9460eaa446d3f7dfb59c408d79cec65a7ec","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"0e7b7131df9efbbc641dfa4dfd266b388acf38942df6680d264c1a305036569d","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100420606720741376","occurred_at":"2026-09-17T03:04:31Z","signature_id":"4b99cbe35f0bd3c6197a97e4f313692f0f80c116d5fba8a8d582ebf622976c32","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298527354654720","occurred_at":"2026-09-16T18:59:25Z","signature_id":"bf7ebbd181ec7aa5127a5794e1778bd8570a9bdf75c05ec0454a8c04349e2cae","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100280971763912704","occurred_at":"2026-09-16T17:49:39Z","signature_id":"e4e6c14d747d24bff47997187743a4306c16de10ca6e0660e678007cc3c9c76a","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"37839b6f366af376a2e2dee46f1b956ca9ee9897a768408f797bc256ab2e722b","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"080a9e45d6f4d800d1c8aca157df99a9c02fa085354fe52205b1fcea2ffcae50","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132038643290112","occurred_at":"2026-09-16T07:57:51Z","signature_id":"fb548ce2c81d7ac7aefb5c63ff2cf1623addd485c6b89a7815a4071a2a0cf5c7","pr_number":6503,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.098901098901099,"anchor_occurrences":8,"after_last_push_count":6,"trend_counts":[0,0,0,2,1,3,1,1],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":8}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T00:38:51Z\n  Ended:       2026-09-21T01:06:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4251392/providers/microsoft.monitor/accounts/hcps-j4251392\", namespace=\"ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4d11d27a-526b-40d4-812b-9a43bae7deb5\"\n  Description: Pod ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-19T01:30:45Z\n  Ended:       2026-09-19T01:58:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8467840-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.52:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8467840/providers/microsoft.monitor/accounts/hcps-j8467840\", namespace=\"ocm-arohcpci01-2stp4bsnt5nt46fh2hgau5todka5ufn5-ingress-cert-le\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"daf93e4d-d43f-49cd-bbc8-4620650d35fc\"\n  Description: Pod ocm-arohcpci01-2stp4bsnt5nt46fh2hgau5todka5ufn5-ingress-cert-le/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T23:52:29Z\n  Ended:       2026-09-19T00:21:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e1a1529f-88f5-4f8c-8503-a2513cf48144\"\n  Description: Pod ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-18T23:53:28Z\n  Ended:       2026-09-19T00:17:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e1a1529f-88f5-4f8c-8503-a2513cf48144\"\n  Description: Pod ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"14896a8781a6fade79c483f0daff72f9c39c1157a6161055b98515a6ef455b60","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101105796258467840","occurred_at":"2026-09-19T00:27:13Z","signature_id":"a33853ee83026eee840aaa65e42f6ac1b995cabd8d6247aef2306bd2a63c0c55","pr_number":7019,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","signature_id":"b0cb1249b73e4e17694cee1167aadd3f0fe5951d24186dea9efdb1984ff09dea","pr_number":7012,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"f71f9d0966fc26ad6ff3cd9a2f086d86818c4234b98d4c027fd22b942a6e7ba2","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","signature_id":"b14a5290094a9e0d25292157800f9158e06c44b980a0e58131fba650ea86837b","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100478808179085312","occurred_at":"2026-09-17T06:55:45Z","signature_id":"62ae640dcc38eb25ad05e49560b842c80c55b99017dce7da9cea433518d78675","pr_number":6988,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","signature_id":"18090d26c3a2666e0dfd30f6dc97088df512304fef09dec58c535d88cec6f5bd","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","signature_id":"2c11c083dc96efa9f51e185909eda32654c395f47b2dbb78eaee76c2573a630d","pr_number":6898,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":8,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.41208791208791207,"anchor_occurrences":8,"after_last_push_count":0,"trend_counts":[0,0,0,3,2,3,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00156c060\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-nzl72g5md8zn\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n      \"name\": \"a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:31:17.188166352Z\",\n      \"endTime\": \"2026-09-18T17:55:21.133690065Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7a39k8no3d3k6o47iqafudkp0ufb still exists (deletion dispatched at 2026-09-18T17:31:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00156c060\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-nzl72g5md8zn\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n      \"name\": \"a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:31:17.188166352Z\",\n      \"endTime\": \"2026-09-18T17:55:21.133690065Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7a39k8no3d3k6o47iqafudkp0ufb still exists (deletion dispatched at 2026-09-18T17:31:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012e61b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-ghmpzk\" in resourcegroup=\"rg-zstream-upgrade-4-20-nzh4xtf2qkhs\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\",\n      \"name\": \"58c027c6-4639-48fa-ace0-f454c7358b62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:43:42.257649954Z\",\n      \"endTime\": \"2026-09-18T18:07:51.061074345Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7ae2rm414v72bb1h0rpr5prt24in still exists (deletion dispatched at 2026-09-18T17:43:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012e61b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-ghmpzk\" in resourcegroup=\"rg-zstream-upgrade-4-20-nzh4xtf2qkhs\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\",\n      \"name\": \"58c027c6-4639-48fa-ace0-f454c7358b62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:43:42.257649954Z\",\n      \"endTime\": \"2026-09-18T18:07:51.061074345Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7ae2rm414v72bb1h0rpr5prt24in still exists (deletion dispatched at 2026-09-18T17:43:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014723f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listg6frkr\" in resourcegroup=\"ea-list-r7jhwkxx4sdv\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n      \"name\": \"9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:26:18.493088022Z\",\n      \"endTime\": \"2026-09-18T17:50:21.112379819Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7eotueh3r9mn8c08fdvae8djl7o8 still exists (deletion dispatched at 2026-09-18T17:26:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014723f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listg6frkr\" in resourcegroup=\"ea-list-r7jhwkxx4sdv\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n      \"name\": \"9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:26:18.493088022Z\",\n      \"endTime\": \"2026-09-18T17:50:21.112379819Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7eotueh3r9mn8c08fdvae8djl7o8 still exists (deletion dispatched at 2026-09-18T17:26:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"7e8e740680a98bd62be8f0f23d65fc40251d4d5f7742370deded696800b1b588","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"9e65f4af22bf7ec9953560fed8bb49590fa82fc63d7f9f6d24d707a1a503f81c","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"a0f110873c9fc7db9f376a400bf0d8b83aa3eddcd644b62e99f4963c1f15aa39","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"56f9b496d774ec73f62dc8457705f5fe010fad5d913e37425e1aeca77033669a","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"e88abeefa8796da83e3bcaacbe5d5b7d5cc078408bb87028a285969eda618a61","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"697bd9b6305361c1b5307d24f6145b0b596cf49460c3ec8e96a3eae19ea908a8","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"6ba2d7f149e18d77de06d3b79ce4ecb80b3fb20da43bf259cd0570583e9d7af6","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"ff4cffb5ab05b8596815bef0f9113f08c5ccdefbdae02ac3f027a27a7d4486ea","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":0.9615384615384616,"anchor_occurrences":7,"after_last_push_count":4,"trend_counts":[0,0,0,2,2,1,0,2],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":7}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T00:59:00Z\n  Ended:       2026-09-21T01:07:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/services-j3696384\", namespace=\"klusterlet-2sv2niah7ifd1s9jttm16f840o7s2bba\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sv2niah7ifd1s9jttm16f840o7s2bba/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-21T01:01:00Z\n  Ended:       2026-09-21T01:07:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/services-j3696384\", namespace=\"klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-21T01:02:03Z\n  Ended:       2026-09-21T01:09:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/services-j3696384\", namespace=\"klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-21T01:02:03Z\n  Ended:       2026-09-21T01:09:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/services-j3696384\", namespace=\"klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 16 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-20T18:23:36Z\n  Ended:       2026-09-20T18:30:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-20T18:23:36Z\n  Ended:       2026-09-20T18:30:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-20T18:23:36Z\n  Ended:       2026-09-20T18:30:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-20T18:23:36Z\n  Ended:       2026-09-20T18:30:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-20T18:23:36Z\n  Ended:       2026-09-20T18:30:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-20T18:23:36Z\n  Ended:       2026-09-20T18:30:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-20T18:23:36Z\n  Ended:       2026-09-20T18:30:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-20T18:23:36Z\n  Ended:       2026-09-20T18:30:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-20T18:24:36Z\n  Ended:       2026-09-20T18:30:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustit9omv6o2votoquk6fobdhva9dc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sustit9omv6o2votoquk6fobdhva9dc/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-20T18:24:36Z\n  Ended:       2026-09-20T18:30:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustit9omv6o2votoquk6fobdhva9dc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sustit9omv6o2votoquk6fobdhva9dc/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-20T18:24:36Z\n  Ended:       2026-09-20T18:30:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustit9omv6o2votoquk6fobdhva9dc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sustit9omv6o2votoquk6fobdhva9dc/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-20T18:24:36Z\n  Ended:       2026-09-20T18:30:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustim6s1ajrlhkslrnmndb00d2cdej\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sustim6s1ajrlhkslrnmndb00d2cdej/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-09-20T18:24:36Z\n  Ended:       2026-09-20T18:30:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustit9omv6o2votoquk6fobdhva9dc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sustit9omv6o2votoquk6fobdhva9dc/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-09-20T18:24:37Z\n  Ended:       2026-09-20T18:30:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustim6s1ajrlhkslrnmndb00d2cdej\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sustim6s1ajrlhkslrnmndb00d2cdej/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-09-20T18:24:37Z\n  Ended:       2026-09-20T18:30:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2susts60dnkbabh5ft9bj4gfhtr5ubsk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2susts60dnkbabh5ft9bj4gfhtr5ubsk/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-09-20T18:26:36Z\n  Ended:       2026-09-20T18:32:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2susu70nvmv9oua374lkat5p6me0e8uj\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2susu70nvmv9oua374lkat5p6me0e8uj/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-19T00:04:52Z\n  Ended:       2026-09-19T00:10:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/services-j8565632\", namespace=\"klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-19T00:04:52Z\n  Ended:       2026-09-19T00:10:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/services-j8565632\", namespace=\"klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"5c8b752be3d2e2575305bb4467922ed388fb9c7274d746c67ce437e73fc6372a","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"8389beb2f2127c28d1c407bb2631de05104a4b0dfdbf0a89c145c62d14049937","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","signature_id":"a46151e80a37451e13b91d43941493c16950cf32997bf773ab888a8c5db06db6","pr_number":7012,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"69b255fd750c54b7a1baf2eca24627dd8e61f4fdef893acd92470a12832db0f6","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"48e28661b4bfc093fce3e4eccd4ac9c8f3996f35cf8b87e4d8ccb1de279e00a8","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"cd9f479af97dbe998875874266d9ea41c8e58990a5cf6ae60b1ca79fabc876a2","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"3b4c127aae1159c832d5991591f49782774b64887255f529b831eb0acc43aae1","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":0.9615384615384616,"anchor_occurrences":7,"after_last_push_count":1,"trend_counts":[0,0,0,5,1,0,0,1],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":7}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-21T00:30:39Z\n  Ended:       2026-09-21T00:36:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-5q82q\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-5q82q has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T02:04:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-6f59c7c94f-hsk8t\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-6f59c7c94f-hsk8t has been in a non-ready state for longer than 5 minutes.","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T21:55:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-dgkdj\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-dgkdj has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T21:55:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-cb282\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-cb282 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T21:55:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-hdnbp\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-hdnbp has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T21:55:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-f58cs\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-f58cs has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"258bf6b8e2e6a15e2ce38412cd2942fddf00da1a19bafe63316ac35a44ba6d7c","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"9a8fc1c68aeeef65c0fa2d8774b771f9f44e0b8d2622880bad833fdcea513992","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100332359231475712","occurred_at":"2026-09-16T21:13:51Z","signature_id":"4cfee31556334d3f7789166186ab36ba11bfce4b40f49a38dd4a3b6152068d51","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100270038647836672","occurred_at":"2026-09-16T17:06:12Z","signature_id":"ce202b7ce3d72dcf4b71c51db3c021d84810e4ab274c64dae9e86007cff0fec7","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"59897c5f85f7527efbbab27b1d42a8c63b62d2873c4723537141e6e4cb09a74c","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161854826352640","occurred_at":"2026-09-16T09:56:19Z","signature_id":"f46fa77bbf961b7174b5b28e58ea0b344fb9d8b591b31027fca64620b0c955e9","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132066686406656","occurred_at":"2026-09-16T07:57:56Z","signature_id":"5c93d0c045993799007f8590467470292210fd23c8f5530b0425e5382165f354","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":0.9615384615384616,"anchor_occurrences":7,"after_last_push_count":5,"trend_counts":[0,0,0,3,0,2,0,2],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":7}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0027fddb0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0027fddb0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00280b770\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00280b770\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001bed8b0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001bed8b0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"0899e170df3dbff91a3d93d67607d26e9d9e1d55e607106e4712f92b087c03fd","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101575058240573440","occurred_at":"2026-09-20T07:31:53Z","signature_id":"73ffdde7c35ef7b4f90a0d9d0266878e677b3024d0af73c677985cd106c56138","pr_number":7025,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101053926852792320","occurred_at":"2026-09-18T21:00:47Z","signature_id":"6566438295d27151c619c08c959579eb7709fb58c49e3c34c87dd1ec1d1e808b","pr_number":6639,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","signature_id":"7193ae4d62017e53f8c9422389a5405738a2e8a036e8555e76291f330d89920e","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298495381475328","occurred_at":"2026-09-16T18:59:17Z","signature_id":"6b84028a0066d5af15d34aa2f3390d5b3b8c5439b097d46537c60eee00e895c1","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100243112008880128","occurred_at":"2026-09-16T15:19:04Z","signature_id":"e0666d1ee60ed5c6d4f663533ea9b4f979d5b8183e061f975c9361ca0cafa025","pr_number":6923,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100237146764873728","occurred_at":"2026-09-16T14:55:30Z","signature_id":"493948c9ce416bdc516799e228ff28ed99f8807ce0868ea1c0984eabe1af5cd2","pr_number":6752,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":7,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.6868131868131868,"anchor_occurrences":7,"after_last_push_count":1,"trend_counts":[0,0,0,6,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a10e80\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8nfj4dg6xpxk\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/063e2ac2-9588-4f9c-82b9-899a69317856\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/063e2ac2-9588-4f9c-82b9-899a69317856\",\n      \"name\": \"063e2ac2-9588-4f9c-82b9-899a69317856\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:48:07.392021884Z\",\n      \"endTime\": \"2026-09-17T03:17:09.326897061Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a10e80\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8nfj4dg6xpxk\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/063e2ac2-9588-4f9c-82b9-899a69317856\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/063e2ac2-9588-4f9c-82b9-899a69317856\",\n      \"name\": \"063e2ac2-9588-4f9c-82b9-899a69317856\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:48:07.392021884Z\",\n      \"endTime\": \"2026-09-17T03:17:09.326897061Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:90]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eaac80\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-5sd5p2zrd22p\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb287309-affe-4cb6-ba60-0ab97101fffb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb287309-affe-4cb6-ba60-0ab97101fffb\",\n      \"name\": \"cb287309-affe-4cb6-ba60-0ab97101fffb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T21:22:15.559693502Z\",\n      \"endTime\": \"2026-09-16T21:51:23.369287505Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:90]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eaac80\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-5sd5p2zrd22p\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb287309-affe-4cb6-ba60-0ab97101fffb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb287309-affe-4cb6-ba60-0ab97101fffb\",\n      \"name\": \"cb287309-affe-4cb6-ba60-0ab97101fffb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T21:22:15.559693502Z\",\n      \"endTime\": \"2026-09-16T21:51:23.369287505Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:584]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101ca40\u003e: \n    failed to create HCP cluster np-downgrade-ptnt6v: failed waiting for cluster=\"np-downgrade-ptnt6v\" in resourcegroup=\"rg-np-downgrade-ptnt6v-s6586bctkvvs\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b81923cc-2583-43fb-a0ea-baa197fb0d6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b81923cc-2583-43fb-a0ea-baa197fb0d6f\",\n      \"name\": \"b81923cc-2583-43fb-a0ea-baa197fb0d6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:07:45.747660708Z\",\n      \"endTime\": \"2026-09-16T15:36:50.286764481Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:584]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101ca40\u003e: \n    failed to create HCP cluster np-downgrade-ptnt6v: failed waiting for cluster=\"np-downgrade-ptnt6v\" in resourcegroup=\"rg-np-downgrade-ptnt6v-s6586bctkvvs\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b81923cc-2583-43fb-a0ea-baa197fb0d6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b81923cc-2583-43fb-a0ea-baa197fb0d6f\",\n      \"name\": \"b81923cc-2583-43fb-a0ea-baa197fb0d6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:07:45.747660708Z\",\n      \"endTime\": \"2026-09-16T15:36:50.286764481Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"d7f93e547e4be2029c411ab0ce7110ad6d4b2c44c4f85c58ccad50e27c80ca41","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"2ffe72f7718510ccd07fc00e17f58bc0013af474b7348754fe62e7d3c4440904","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"f53d0a32dc007711392e3ac9a6346e4050a79f13a86cf1b09a36647de1c455fb","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"55ad0310438d8f56a4a299f6bab3b8eead74cd3a5f7c47eb326a8464bccf50a4","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"91a400cbca3134720c48834459e3002d3cce5a22f2d380c716d75f65f9d75177","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"b82de203b62ff6756c30d031383d9f2c1cda2b52933916b1090a4b903f7e89b9","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"25f5f479c873e8672b346f86e1757b6d6cdd4c81dcdc1af82909656949ba3709","pr_number":6879,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"52bfcbbc720001a7392b6a1e08f396579b61443952ed16c2c80c98fe6498c690","failure_pattern":"alert [svc] userJourneyFrontendLatencyP996h30m fired","search_query":"alert [svc] userJourneyFrontendLatencyP996h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP996h30m does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.8241758241758241,"anchor_occurrences":6,"after_last_push_count":4,"trend_counts":[0,0,0,6,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP996h30m does not fire","occurrences":6}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T09:29:21Z\n  Ended:       2026-09-16T09:49:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatencyP996h30m\", cluster=\"ci01-j5155200-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5155200/providers/microsoft.monitor/accounts/services-j5155200\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n  Description: Frontend p99 latency on cluster ci01-j5155200-svc is above 1s on both the 6h and 30m windows (p99 \u003c 1s ARM sync SLO miss).","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T09:46:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatencyP996h30m\", cluster=\"ci01-j3290112-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3290112/providers/microsoft.monitor/accounts/services-j3290112\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n  Description: Frontend p99 latency on cluster ci01-j3290112-svc is above 1s on both the 6h and 30m windows (p99 \u003c 1s ARM sync SLO miss).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T08:53:50Z\n  Ended:       2026-09-16T09:06:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatencyP996h30m\", cluster=\"ci01-j9843328-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9843328/providers/microsoft.monitor/accounts/services-j9843328\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n  Description: Frontend p99 latency on cluster ci01-j9843328-svc is above 1s on both the 6h and 30m windows (p99 \u003c 1s ARM sync SLO miss)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132075855155200","occurred_at":"2026-09-16T07:57:59Z","signature_id":"4e60a01b4827c68ff57b07edc9eab558ce75aad5112478ce2737bee6696f871b","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132038643290112","occurred_at":"2026-09-16T07:57:51Z","signature_id":"9daba40c60f37faf2af38abc3e3474d5917ef206c95bff1353f0bdaf3df9ac3b","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"cbb7931aab2d0ed1916b8d84363dfe1ffc05cf35d9ce851f977710db9cdf561f","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6971/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100127403039789056","occurred_at":"2026-09-16T07:39:25Z","signature_id":"02815353fa0a66ddf20b88ae30c92eac6b08249d84094735fa13cf0c3952861f","pr_number":6971,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6923/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100110113200148480","occurred_at":"2026-09-16T06:30:43Z","signature_id":"d87f770fb784724db359656072889a1f3015ce39f593db987908535455d1ee42","pr_number":6923,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100104517457547264","occurred_at":"2026-09-16T06:08:29Z","signature_id":"d21b60b6e4bcd8f18d5041389daab6c5b24d976173fd091a6cd2e7e5a098f762","pr_number":6894,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.8241758241758241,"anchor_occurrences":6,"after_last_push_count":4,"trend_counts":[0,0,0,2,1,3,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":6}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001a0ed40\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59914-\u003e172.30.0.10:53: read: connection refused\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001a0ed40\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59914-\u003e172.30.0.10:53: read: connection refused\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.002s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000c8c390\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:35668-\u003e172.30.0.10:53: read: connection refused\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.002s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000c8c390\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:35668-\u003e172.30.0.10:53: read: connection refused\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0018c70e0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59154-\u003e172.30.0.10:53: read: connection refused\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0018c70e0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59154-\u003e172.30.0.10:53: read: connection refused\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7003/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100859139801485312","occurred_at":"2026-09-18T08:07:05Z","signature_id":"4553dd78f39dc989ce886fb5e82604c7aa8be953927622430a5564e21a9cc7f0","pr_number":7003,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827581187100672","occurred_at":"2026-09-18T06:01:39Z","signature_id":"25746549fcd7e3d493a2e9010aa4c2f674167bdbce01dd388ff30ebecae482d7","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780868975988736","occurred_at":"2026-09-18T02:56:04Z","signature_id":"3f4f2a1f835d80221855e8810686dfbc430d2c91ebb857edce8074813fea620c","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"41e17c7a4eecb6c1ed4ee23560b530d6b6a42361cc6e3ef0250dbdef28f6d545","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100294017282478080","occurred_at":"2026-09-16T18:41:29Z","signature_id":"30b7eed5c08665fa2cef128c104b6bc116a828b17cff3a0328ac6b1a7aaf5b24","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","signature_id":"e12e1b463aa599590cba976eac3785b82a07ad6fd298aa2e1664ebd292e43daa","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"40f7a558a9f14e3ad84a0dc62150fb9e1dcec198cbfa1c00cbbb104cbc63f3a7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":6,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":6,"after_last_push_count":0,"trend_counts":[0,0,0,6,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0028a4438\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-j5zkl24x6bwt\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecb9f85-876c-467d-91c9-980ce11c9827\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecb9f85-876c-467d-91c9-980ce11c9827\",\n      \"name\": \"5ecb9f85-876c-467d-91c9-980ce11c9827\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:49:19.070350042Z\",\n      \"endTime\": \"2026-09-16T10:13:22.487477443Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0028a4438\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-j5zkl24x6bwt\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecb9f85-876c-467d-91c9-980ce11c9827\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecb9f85-876c-467d-91c9-980ce11c9827\",\n      \"name\": \"5ecb9f85-876c-467d-91c9-980ce11c9827\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:49:19.070350042Z\",\n      \"endTime\": \"2026-09-16T10:13:22.487477443Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00153e0f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-9dwsgxpprw8c\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\",\n      \"name\": \"9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:44:48.885828252Z\",\n      \"endTime\": \"2026-09-16T10:08:52.4506315Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00153e0f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-9dwsgxpprw8c\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\",\n      \"name\": \"9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:44:48.885828252Z\",\n      \"endTime\": \"2026-09-16T10:08:52.4506315Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000012f78\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-g9bvp2\" in resourcegroup=\"rg-candidate-4-20-pmdbfsk6v6zd\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/834c2939-b42e-48bd-b340-960d8826a429\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/834c2939-b42e-48bd-b340-960d8826a429\",\n      \"name\": \"834c2939-b42e-48bd-b340-960d8826a429\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:49:50.424611242Z\",\n      \"endTime\": \"2026-09-16T10:13:52.492147666Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000012f78\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-g9bvp2\" in resourcegroup=\"rg-candidate-4-20-pmdbfsk6v6zd\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/834c2939-b42e-48bd-b340-960d8826a429\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/834c2939-b42e-48bd-b340-960d8826a429\",\n      \"name\": \"834c2939-b42e-48bd-b340-960d8826a429\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:49:50.424611242Z\",\n      \"endTime\": \"2026-09-16T10:13:52.492147666Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"18c69429807d38cfdd701e251877ce552b6dddaac55af8cb470eb1279a95b526","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"232d76b4c8604eadbe09c019eea03e4b35e8f44b8a88fbcca5afcc985e967ce4","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"9410511803cbf1e3c159d5decef593399c3f8d02958271b7b1f469d232f45e6e","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"9635d65ddbc8d8183553acfbe1c7e3ebf7cfb49f71fb8721193391b355cf2fbd","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"ba6d9f57396d35aba73dd64e7c229f8d7cbb408e58e239c7c2b2065df4e28656","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"d539c69fd361c2a857c348ec75ef524c1710267ecef6f193ceb123bc3e8bce18","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.6868131868131868,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,0,5,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":5}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T22:06:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.104:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1475712/providers/microsoft.monitor/accounts/services-j1475712\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-59mfc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T22:06:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.104:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1475712/providers/microsoft.monitor/accounts/services-j1475712\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-59mfc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:55:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7836672/providers/microsoft.monitor/accounts/services-j7836672\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-zdqrc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:55:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7836672/providers/microsoft.monitor/accounts/services-j7836672\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-zdqrc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T16:45:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T16:45:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T16:46:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T16:46:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100332359231475712","occurred_at":"2026-09-16T21:13:51Z","signature_id":"617d8bea63f7c0a8e5aa65e252453013f2f87eaabc747dea5d193edf0ea92b6a","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100270038647836672","occurred_at":"2026-09-16T17:06:12Z","signature_id":"659a105790bd191ed63ba5bb7e543dad17b2528556030da7e70281aa38365890","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"8cefc20a001466adc6a61bb43c8407f0170786f35f4348a8d83cb4fc19e10669","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161854826352640","occurred_at":"2026-09-16T09:56:19Z","signature_id":"73664eccd132c9d84475a61b44857f75314a6635b79c98d2ae19c6ecf756089a","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132066686406656","occurred_at":"2026-09-16T07:57:56Z","signature_id":"358902433200cc0a70d0c371110e19da053abecdbf25175a02fa1df21c363226","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.6868131868131868,"anchor_occurrences":5,"after_last_push_count":2,"trend_counts":[0,0,0,2,0,3,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.009s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.009s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.000s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.000s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101053926852792320","occurred_at":"2026-09-18T21:00:47Z","signature_id":"453c681e05619434e546781762e4d8f81384d3fa3b09f4cc9d65ac2b7dc22b1d","pr_number":6639,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827581187100672","occurred_at":"2026-09-18T06:01:39Z","signature_id":"ee111baab05f6b8ee26f353e865437e1bffda7140d9fc154398f141cd776e6f7","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6813/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815351901065216","occurred_at":"2026-09-18T05:13:05Z","signature_id":"453c681e05619434e546781762e4d8f81384d3fa3b09f4cc9d65ac2b7dc22b1d","pr_number":6813,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","signature_id":"ee111baab05f6b8ee26f353e865437e1bffda7140d9fc154398f141cd776e6f7","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100164075236364288","occurred_at":"2026-09-16T10:05:09Z","signature_id":"186ac175cc37e33f1eaa9e81000b9ad2fcf1670566cc6849f36ba3c782d224b2","pr_number":6688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b414f524aec7c4ea3c6d464450b9af6f734740715a61aff1c3a8f53547f62641","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":5,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.5494505494505495,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,0,3,2,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000648280\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qghsblcj4pgk\" to finish creating: GET https://rp.j4009472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\",\n      \"name\": \"f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T23:29:11.075687263Z\",\n      \"endTime\": \"2026-09-17T23:58:14.591633856Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000648280\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qghsblcj4pgk\" to finish creating: GET https://rp.j4009472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\",\n      \"name\": \"f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T23:29:11.075687263Z\",\n      \"endTime\": \"2026-09-17T23:58:14.591633856Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015ec480\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-4llzpkxqdt5h\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/98da8819-6ed6-431a-b25f-78eab8ef8dd9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/98da8819-6ed6-431a-b25f-78eab8ef8dd9\",\n      \"name\": \"98da8819-6ed6-431a-b25f-78eab8ef8dd9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:12:29.574806708Z\",\n      \"endTime\": \"2026-09-17T13:41:32.177488403Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015ec480\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-4llzpkxqdt5h\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/98da8819-6ed6-431a-b25f-78eab8ef8dd9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/98da8819-6ed6-431a-b25f-78eab8ef8dd9\",\n      \"name\": \"98da8819-6ed6-431a-b25f-78eab8ef8dd9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:12:29.574806708Z\",\n      \"endTime\": \"2026-09-17T13:41:32.177488403Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e4c880\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7vfjq4g5mpkz\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\",\n      \"name\": \"fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:25:02.43485078Z\",\n      \"endTime\": \"2026-09-16T15:54:04.537444594Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e4c880\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7vfjq4g5mpkz\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\",\n      \"name\": \"fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:25:02.43485078Z\",\n      \"endTime\": \"2026-09-16T15:54:04.537444594Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"3952b4207ca326cafd047aae37e1541b9c1921b3b3a1de670914b6430814079b","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"d7b3a6738a3142cf28e35b425618e47c2d78a5ec741df6019ce6ae2b560d9004","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"5f788a7a822406c498eab147df2fe3fad0c3f67d39005f710714ed9d67ac601d","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"976862ec83bbad292378adfc6b6876d3447587a76e0f813a8827a52bd7db4bbc","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"4c23de01cdd9b886d00d4c9a0959f33955c73d1a556a45314d353da1da388706","pr_number":6976,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"575b0de6701cb1cdcdd3b5c4e6523248471d3aa67c3c0c1fed243233e4014b33","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Monitor","search_query":"ERROR CODE: ResourceGroupNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"AMA Metrics service metrics from SVC and MGMT clusters should be present in Azure Monitor","occurrences":5,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.41208791208791207,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,0,4,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"AMA Metrics service metrics from SVC and MGMT clusters should be present in Azure Monitor","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-j4886144 in resource group hcp-underlay-ci01-j4886144\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b90a00\u003e: \n    failed to get workspace services-j4886144: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4886144/providers/Microsoft.Monitor/accounts/services-j4886144\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j4886144' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-j4886144 in resource group hcp-underlay-ci01-j4886144\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b90a00\u003e: \n    failed to get workspace services-j4886144: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4886144/providers/Microsoft.Monitor/accounts/services-j4886144\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j4886144' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:74]: failed to look up HCP Prometheus endpoint\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00072e820\u003e: \n    failed to get workspace hcps-j3128064: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3128064/providers/Microsoft.Monitor/accounts/hcps-j3128064\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j3128064' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:74]: failed to look up HCP Prometheus endpoint\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00072e820\u003e: \n    failed to get workspace hcps-j3128064: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3128064/providers/Microsoft.Monitor/accounts/hcps-j3128064\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j3128064' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-j3128064 in resource group hcp-underlay-ci01-j3128064\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010489e0\u003e: \n    failed to get workspace services-j3128064: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3128064/providers/Microsoft.Monitor/accounts/services-j3128064\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j3128064' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-j3128064 in resource group hcp-underlay-ci01-j3128064\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010489e0\u003e: \n    failed to get workspace services-j3128064: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3128064/providers/Microsoft.Monitor/accounts/services-j3128064\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j3128064' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"1e24a52fcfa8fdc207f5da162e136847f4db22f73c90f28694d25d387bc9480d","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","signature_id":"1d95e250094820742e5d106c9a6198395eff4115c52cfbf3176662d0996cbb8f","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","signature_id":"2300bcf0e83155e35375beb6ab2cd2dd1e60df39d954749c7fd210ed41e13cde","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"4a840acfa7ba40488e61bba537bd235d758cbe0b71e71c1cdf5af1d8876188b2","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"e60b8a3f0f846e4caf38925534828b81ff9b59a90b098e189f25e9987eb8d737","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fe6b01560ac49646fd0a46c48f672cf2fc0ebbf844ea4576bb0cab7a46df7ce8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":5,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.27472527472527475,"anchor_occurrences":5,"after_last_push_count":1,"trend_counts":[0,0,0,4,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f90a40\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8x87dzwxvb96\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3edfae6e-fb2d-4e52-a859-1500f9def9f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3edfae6e-fb2d-4e52-a859-1500f9def9f6\",\n      \"name\": \"3edfae6e-fb2d-4e52-a859-1500f9def9f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:04:32.691734276Z\",\n      \"endTime\": \"2026-09-17T02:33:33.968010236Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f90a40\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8x87dzwxvb96\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3edfae6e-fb2d-4e52-a859-1500f9def9f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3edfae6e-fb2d-4e52-a859-1500f9def9f6\",\n      \"name\": \"3edfae6e-fb2d-4e52-a859-1500f9def9f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:04:32.691734276Z\",\n      \"endTime\": \"2026-09-17T02:33:33.968010236Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:103]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001f3c120\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sxmn5zhcsvjn\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a019abb2-ae16-4218-9707-6f7dc73a3eb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a019abb2-ae16-4218-9707-6f7dc73a3eb3\",\n      \"name\": \"a019abb2-ae16-4218-9707-6f7dc73a3eb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:08:36.527287031Z\",\n      \"endTime\": \"2026-09-16T15:37:40.20476418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:103]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001f3c120\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sxmn5zhcsvjn\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a019abb2-ae16-4218-9707-6f7dc73a3eb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a019abb2-ae16-4218-9707-6f7dc73a3eb3\",\n      \"name\": \"a019abb2-ae16-4218-9707-6f7dc73a3eb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:08:36.527287031Z\",\n      \"endTime\": \"2026-09-16T15:37:40.20476418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--7tj5b6r9d28k/cluster-hs-pre-rwztb2 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ee48a0\u003e: \n    failed to create HCP cluster cluster-hs-pre-rwztb2: failed waiting for cluster=\"cluster-hs-pre-rwztb2\" in resourcegroup=\"rg-hs-pre--7tj5b6r9d28k\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0015f221-0420-4231-b96b-76ce90032f9d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0015f221-0420-4231-b96b-76ce90032f9d\",\n      \"name\": \"0015f221-0420-4231-b96b-76ce90032f9d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:07:38.092292569Z\",\n      \"endTime\": \"2026-09-16T15:36:40.334174064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--7tj5b6r9d28k/cluster-hs-pre-rwztb2 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ee48a0\u003e: \n    failed to create HCP cluster cluster-hs-pre-rwztb2: failed waiting for cluster=\"cluster-hs-pre-rwztb2\" in resourcegroup=\"rg-hs-pre--7tj5b6r9d28k\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0015f221-0420-4231-b96b-76ce90032f9d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0015f221-0420-4231-b96b-76ce90032f9d\",\n      \"name\": \"0015f221-0420-4231-b96b-76ce90032f9d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:07:38.092292569Z\",\n      \"endTime\": \"2026-09-16T15:36:40.334174064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"9020e6f833dc38c47ccef3d4182afa72cebc722f85d46ac12ebd56299acce896","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"0bf39d4aab091ac51c471f05a7fa67a4d591d3eb93555a71c26170936f523c5f","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"a64a06ed18c7d9e35b003b4339972fe04157c007a8cd68d29875724bb6a184d4","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"b2b0b3c510ba3d2ec5bcda27a848edbf0f6e4130232ba677304593697d08e0b2","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"e41f400f85ccdcde13c1223820f617791e8453915fe2b4d9d6f910f58c9aac3e","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.5494505494505495,"anchor_occurrences":4,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[0,0,0,2,1,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:183]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:183]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410","failure_pattern":"ERROR CODE: DeploymentStackDeploymentFailed","search_query":"ERROR CODE: DeploymentStackDeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.5494505494505495,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,0,2,0,2,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","occurrences":4}],"full_error_samples":["time=2026-09-18T21:50:35.372Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n  Kind: ARMStack\\n  Template: templates/monitoring.bicep\\n  Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-09-18T21:51:25.579Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/04392d3b-593d-406d-ae87-240d0d6c1c70\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/04392d3b-593d-406d-ae87-240d0d6c1c70\\\",\\n  \\\"name\\\": \\\"04392d3b-593d-406d-ae87-240d0d6c1c70\\\",\\n  \\\"status\\\": \\\"failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n    \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'aad2e336-6c43-473c-997c-4a60a482583a'.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4937088/providers/Microsoft.Resources/deployments/y5flko475afn22dntfyewzfj7gfkbiwdmj7ezjquufwse-26091821etad3\\\",\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n        \\\"details\\\": [\\n          {\\n            \\\"code\\\": \\\"BadRequest\\\",\\n            \\\"message\\\": \\\"Property name: Rules[0].Expression, Attempted value: null, Error: 1:237: parse error: unknown escape sequence U+002E '.' Activity ID: b8850252-9aac-448f-9581-e29b2b8a10a8.\\\"\\n          }\\n        ]\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-18T15:40:00.669Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n  Kind: ARMStack\\n  Template: templates/monitoring.bicep\\n  Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-09-18T15:42:51.615Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/eastus2/deploymentStackOperationStatus/326eddf6-9965-4299-86de-7e1eb754418c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/eastus2/deploymentStackOperationStatus/326eddf6-9965-4299-86de-7e1eb754418c\\\",\\n  \\\"name\\\": \\\"326eddf6-9965-4299-86de-7e1eb754418c\\\",\\n  \\\"status\\\": \\\"failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n    \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'e78fa208-8c43-45b6-a80c-57ad93c6a044'.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4742016/providers/Microsoft.Resources/deployments/ka7tdccaiavlrtibmy3y5s7tfhcs6gpzxc6p4xe5pchjy-26091815bptz8\\\",\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n        \\\"details\\\": [\\n          {\\n            \\\"code\\\": \\\"BadRequest\\\",\\n            \\\"message\\\": \\\"Property name: Rules[0].Expression, Attempted value: null, Error: 1:237: parse error: unknown escape sequence U+002E '.' Activity ID: d9b78f81-a4e0-4b15-9e4d-75e6f5c342e3.\\\"\\n          }\\n        ]\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-16T18:22:02.494Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n  Kind: ARMStack\\n  Template: templates/monitoring.bicep\\n  Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-09-16T18:22:51.675Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/canadacentral/deploymentStackOperationStatus/1df7c411-d9ac-443d-ac5d-b95681f8419b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/canadacentral/deploymentStackOperationStatus/1df7c411-d9ac-443d-ac5d-b95681f8419b\\\",\\n  \\\"name\\\": \\\"1df7c411-d9ac-443d-ac5d-b95681f8419b\\\",\\n  \\\"status\\\": \\\"failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n    \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'd478c792-be07-45be-a743-97b4e94abf15'.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5982976/providers/Microsoft.Resources/deployments/nbyngfosfawbsx5vtxb2uo6lmfim2rbaopp3emm2zwlmm-260916186gla8\\\",\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n        \\\"details\\\": [\\n          {\\n            \\\"code\\\": \\\"BadRequest\\\",\\n            \\\"message\\\": \\\"Property name: Rules[0].Expression, Attempted value: null, Error: 1:237: parse error: unknown escape sequence U+002E '.' Activity ID: 95e8cbf4-5734-47ec-859e-d6ed81e0056f.\\\"\\n          }\\n        ]\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101062836254937088","occurred_at":"2026-09-18T21:36:30Z","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","pr_number":6965,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100966207174742016","occurred_at":"2026-09-18T15:08:20Z","signature_id":"e4c3fb1587b09488a2265155d7a6787ca94e35b729bce621d58ebb8765ea7ef1","pr_number":6965,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100283069305982976","occurred_at":"2026-09-16T17:57:59Z","signature_id":"2a5637582a78b2337c484f5a7a9ff7f9d84aa702c7b086e89182f1549eb13d57","pr_number":6965,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100265336732389376","occurred_at":"2026-09-16T16:47:31Z","signature_id":"2a5637582a78b2337c484f5a7a9ff7f9d84aa702c7b086e89182f1549eb13d57","pr_number":6965,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.5494505494505495,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,0,4,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":4}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T22:05:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j1475712-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j1475712-svc has been unreachable for 15 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:55:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j7836672-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j7836672-svc has been unreachable for 15 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T10:51:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j6352640-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j6352640-svc has been unreachable for 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100332359231475712","occurred_at":"2026-09-16T21:13:51Z","signature_id":"9b1a4c0162aa96d1ba2cfc1a58fe8450105df2f95a84e56b0c0206685fdb86fb","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100270038647836672","occurred_at":"2026-09-16T17:06:12Z","signature_id":"c1988f523c9b1951a937f22e869077937abcdb055a057f4fd9deb538ebdeeab8","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161854826352640","occurred_at":"2026-09-16T09:56:19Z","signature_id":"40d8f345b61cc5a9851d9600ef3889f30540178c4979cc679d48f241649fe011","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132066686406656","occurred_at":"2026-09-16T07:57:56Z","signature_id":"cdf1b6098bdd9c6cf95b2228b3d492b2e30cb0b5e195c0384f719de68055f224","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gn4q6q4pqf7b\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.5494505494505495,"anchor_occurrences":4,"after_last_push_count":0,"also_in":["PROD"],"trend_counts":[0,0,0,1,1,1,1,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001364120\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-49gn4qjgmx5r\" to finish creating: GET https://rp.j7481216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d405444-e5c6-413b-8e97-5fb154fce110\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d405444-e5c6-413b-8e97-5fb154fce110\",\n      \"name\": \"5d405444-e5c6-413b-8e97-5fb154fce110\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-19T11:23:33.676780621Z\",\n      \"endTime\": \"2026-09-19T11:42:34.970958081Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001364120\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-49gn4qjgmx5r\" to finish creating: GET https://rp.j7481216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d405444-e5c6-413b-8e97-5fb154fce110\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d405444-e5c6-413b-8e97-5fb154fce110\",\n      \"name\": \"5d405444-e5c6-413b-8e97-5fb154fce110\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-19T11:23:33.676780621Z\",\n      \"endTime\": \"2026-09-19T11:42:34.970958081Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0031a8160\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-p9htbjsmj226\" to finish creating: GET https://rp.j6818048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\",\n      \"name\": \"1616a745-fc49-408c-a84f-41679708fa49\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T09:27:23.7095894Z\",\n      \"endTime\": \"2026-09-18T09:28:03.46142651Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0031a8160\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-p9htbjsmj226\" to finish creating: GET https://rp.j6818048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\",\n      \"name\": \"1616a745-fc49-408c-a84f-41679708fa49\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T09:27:23.7095894Z\",\n      \"endTime\": \"2026-09-18T09:28:03.46142651Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0009e8ec0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qbcpd5p47nxj\" to finish creating: GET https://rp.j4979456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3bad1b6-65e5-4da7-a96f-1c7006af3c80\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3bad1b6-65e5-4da7-a96f-1c7006af3c80\",\n      \"name\": \"f3bad1b6-65e5-4da7-a96f-1c7006af3c80\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:12:19.756903274Z\",\n      \"endTime\": \"2026-09-17T13:12:54.770011764Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0009e8ec0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qbcpd5p47nxj\" to finish creating: GET https://rp.j4979456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3bad1b6-65e5-4da7-a96f-1c7006af3c80\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3bad1b6-65e5-4da7-a96f-1c7006af3c80\",\n      \"name\": \"f3bad1b6-65e5-4da7-a96f-1c7006af3c80\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:12:19.756903274Z\",\n      \"endTime\": \"2026-09-17T13:12:54.770011764Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101255587097481216","occurred_at":"2026-09-19T10:22:26Z","signature_id":"0d3ffcb6a10b4863e652f98c9e6f1249ec0ee51db3fe0b606bafa58f78f7be8b","pr_number":4878,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","signature_id":"1ff84f075d12052118fcca1c651ed549b6c5f0208069e0615f46732cae26b19e","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554708664979456","occurred_at":"2026-09-17T11:57:12Z","signature_id":"7dda8cc2984aa1b8ad7daf00d042d0064ecf3188c6f0d908489ee4571d215e40","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"162026bfd655fc4db32e7049c532e8d81f8bd34a8e4e6625145cd5a31c7568a3","pr_number":6976,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c71ebc2a93d9fc10a75dcaf39a2b1325de6379c3891238b64d1e4e8f4281df9a","failure_pattern":"failed to get kusto cluster \u003ckusto-cluster\u003e from config: configuration: key kusto not found","search_query":"failed to get kusto cluster name from config: configuration: key kusto not found","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.5494505494505495,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,0,2,2,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000aac060\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000aac060\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0018b8060\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0018b8060\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0012e60a0\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0012e60a0\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100556682735128576","occurred_at":"2026-09-17T12:05:14Z","signature_id":"99c5a5b19f403073edd6e8406f2272a3fda7ff44e9a289f5c999503a39917368","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"daa3fa7c154560025dfba9dba52f2111304f2d047aafee6e951b7d85e52a0fdc","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","signature_id":"9b7d539624448efd4996e6b535e22c740df41bfa52b50eab26733ce795540f15","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"12729704797975e796e396a1c7d5eb085f228e272a9762c5cf68851df1b74ece","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.41208791208791207,"anchor_occurrences":4,"after_last_push_count":1,"trend_counts":[0,0,0,4,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a9c270\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-fqmsfk\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-lqr7gtcm4srq\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\",\n      \"name\": \"3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:13:26.770029811Z\",\n      \"endTime\": \"2026-09-16T17:37:32.768384818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j3fnpr9m9cqfjaohaiq73k6sjvpi still exists (deletion dispatched at 2026-09-16T17:13:27Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a9c270\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-fqmsfk\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-lqr7gtcm4srq\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\",\n      \"name\": \"3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:13:26.770029811Z\",\n      \"endTime\": \"2026-09-16T17:37:32.768384818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j3fnpr9m9cqfjaohaiq73k6sjvpi still exists (deletion dispatched at 2026-09-16T17:13:27Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014240f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xbmtbzvlpclx\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d85e799c-786e-4172-95d1-86693923282f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d85e799c-786e-4172-95d1-86693923282f\",\n      \"name\": \"d85e799c-786e-4172-95d1-86693923282f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:32:57.751739191Z\",\n      \"endTime\": \"2026-09-16T17:57:03.956555792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j51n0bvbqmvanc2bru5bjg1kuddr still exists (deletion dispatched at 2026-09-16T17:32:58Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014240f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xbmtbzvlpclx\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d85e799c-786e-4172-95d1-86693923282f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d85e799c-786e-4172-95d1-86693923282f\",\n      \"name\": \"d85e799c-786e-4172-95d1-86693923282f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:32:57.751739191Z\",\n      \"endTime\": \"2026-09-16T17:57:03.956555792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j51n0bvbqmvanc2bru5bjg1kuddr still exists (deletion dispatched at 2026-09-16T17:32:58Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000012e40\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-wbh9jqvv44p7\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9db0308e-c5ac-4693-b367-d76dc77af863\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9db0308e-c5ac-4693-b367-d76dc77af863\",\n      \"name\": \"9db0308e-c5ac-4693-b367-d76dc77af863\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:29:44.443897591Z\",\n      \"endTime\": \"2026-09-16T15:53:44.471117038Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss64i7i766p1sol3r0dalpub5p7l53h still exists (deletion dispatched at 2026-09-16T15:29:46Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000012e40\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-wbh9jqvv44p7\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9db0308e-c5ac-4693-b367-d76dc77af863\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9db0308e-c5ac-4693-b367-d76dc77af863\",\n      \"name\": \"9db0308e-c5ac-4693-b367-d76dc77af863\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:29:44.443897591Z\",\n      \"endTime\": \"2026-09-16T15:53:44.471117038Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss64i7i766p1sol3r0dalpub5p7l53h still exists (deletion dispatched at 2026-09-16T15:29:46Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"4b15da3ce1591e33d2696652650de2d0cdc6c841165c33a2a923881126078487","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"4f20aa230f0afb2f03a0196f22a9def6c79b679a7bc9bc79347900fd06c241a8","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"b79349f0a2b72f52e04da9ac4e8ef7179dd24862a80796349270662d77641269","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222891458039808","occurred_at":"2026-09-16T13:58:52Z","signature_id":"8491d54e1e6f9d99de4d3a50c4e2d20f3eb57611f24ff8bb3d09a72d4b3c8424","pr_number":6952,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.41208791208791207,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[0,0,0,0,1,1,0,1],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ef5080\u003e: \n    DNS for route host agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ef5080\u003e: \n    DNS for route host agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-q78vhx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f621e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-q78vhx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f621e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-prgsd6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106cb20\u003e: \n    DNS for route host agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-prgsd6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106cb20\u003e: \n    DNS for route host agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","signature_id":"571f2e2e0cd4941242495f0f83848c90e92c491345357a2fc6f06f0266aa0a09","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"803dd3435e0189d03ce01d4d7de3f8e741a5ee356c8001e19b84595bbaa71298","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524951428665344","occurred_at":"2026-09-17T09:59:04Z","signature_id":"b9c07fb6e82776d012cae86cfc979add2b4b19f7b344021874689105ee18c987","pr_number":5724,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a6bab59ff05e72a771d612966dd1e7024c1a3d21427661ca54c8003eb2c7e563","failure_pattern":"ERROR CODE: DeploymentFailed; detail code GetMSICredentialError; detail message Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.41208791208791207,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[0,0,0,1,0,2,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1}],"full_error_samples":["time=2026-09-18T03:37:37.864Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-18T03:37:38.393Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-18T03:37:39.053Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=9d03ef43fb389999fc408f90bd02e6c94ce2721f8e17a8400c8a777195fbc298 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1401344-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9d03ef43fb389999fc408f90bd02e6c94ce2721f8e17a8400c8a777195fbc298\ntime=2026-09-18T03:40:39.648Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1401344-mgmt-1/providers/Microsoft.Resources/deployments/9d03ef43fb389999fc408f90bd02e6c94ce2721f8e17a8400c8a777195fbc298/operationStatuses/08584119042267536349\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1401344-mgmt-1/providers/Microsoft.Resources/deployments/cluster-sr7huchijtjyg\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1401344-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j1401344-mgmt-1\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"GetMSICredentialError\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-18T01:22:54.255Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-18T01:22:54.877Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-18T01:22:55.881Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=20d345ced78ed93f1fcc83ca672f15621e19f0c6572d4980fecd14d77cfa1a28 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9648512-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F20d345ced78ed93f1fcc83ca672f15621e19f0c6572d4980fecd14d77cfa1a28\ntime=2026-09-18T01:24:56.385Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9648512-mgmt-2/providers/Microsoft.Resources/deployments/20d345ced78ed93f1fcc83ca672f15621e19f0c6572d4980fecd14d77cfa1a28/operationStatuses/08584119123100595444\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9648512-mgmt-2/providers/Microsoft.Resources/deployments/cluster-zhmgezzmxfkbc\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9648512-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j9648512-mgmt-2\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"GetMSICredentialError\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-16T12:27:55.094Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-09-16T12:27:56.479Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-09-16T12:27:57.833Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=b489aeac3f7b0efc7498443b42a85f2f65ee62ab4f82355239f486d3dbc78483 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5020544-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fb489aeac3f7b0efc7498443b42a85f2f65ee62ab4f82355239f486d3dbc78483\ntime=2026-09-16T12:31:28.399Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5020544-svc/providers/Microsoft.Resources/deployments/b489aeac3f7b0efc7498443b42a85f2f65ee62ab4f82355239f486d3dbc78483/operationStatuses/08584120452084424414\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5020544-svc/providers/Microsoft.Resources/deployments/cluster-6sk23df5akobi\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5020544-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j5020544-svc\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"GetMSICredentialError\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6984/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784773881401344","occurred_at":"2026-09-18T03:11:35Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6984,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100751960339648512","occurred_at":"2026-09-18T01:00:47Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100192568775020544","occurred_at":"2026-09-16T11:58:22Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6964,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.41208791208791207,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[0,0,0,2,0,0,0,1],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001366090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-fxwzs2\" in resourcegroup=\"rg-np-downgrade-fxwzs2-xgl2x967w267\" to finish deleting: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf94f468-3ed1-44cc-a51b-69992a140dfb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf94f468-3ed1-44cc-a51b-69992a140dfb\",\n      \"name\": \"bf94f468-3ed1-44cc-a51b-69992a140dfb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T01:01:37.424253631Z\",\n      \"endTime\": \"2026-09-21T01:25:46.248853963Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2nnff2dqdtbc8d6g85073gq2fnh75 still exists (deletion dispatched at 2026-09-21T01:01:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001366090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-fxwzs2\" in resourcegroup=\"rg-np-downgrade-fxwzs2-xgl2x967w267\" to finish deleting: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf94f468-3ed1-44cc-a51b-69992a140dfb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf94f468-3ed1-44cc-a51b-69992a140dfb\",\n      \"name\": \"bf94f468-3ed1-44cc-a51b-69992a140dfb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-21T01:01:37.424253631Z\",\n      \"endTime\": \"2026-09-21T01:25:46.248853963Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2nnff2dqdtbc8d6g85073gq2fnh75 still exists (deletion dispatched at 2026-09-21T01:01:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019540a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-kttjnq\" in resourcegroup=\"rg-np-downgrade-kttjnq-75tphq26mtkt\" to finish deleting: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/941782dd-efae-4093-940b-ca21585fc901\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/941782dd-efae-4093-940b-ca21585fc901\",\n      \"name\": \"941782dd-efae-4093-940b-ca21585fc901\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:46:21.836683685Z\",\n      \"endTime\": \"2026-09-16T21:10:27.160876667Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssarr47iva7e9en1e12mbat1fvbbvpq still exists (deletion dispatched at 2026-09-16T20:46:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019540a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-kttjnq\" in resourcegroup=\"rg-np-downgrade-kttjnq-75tphq26mtkt\" to finish deleting: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/941782dd-efae-4093-940b-ca21585fc901\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/941782dd-efae-4093-940b-ca21585fc901\",\n      \"name\": \"941782dd-efae-4093-940b-ca21585fc901\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:46:21.836683685Z\",\n      \"endTime\": \"2026-09-16T21:10:27.160876667Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssarr47iva7e9en1e12mbat1fvbbvpq still exists (deletion dispatched at 2026-09-16T20:46:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001654180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-vwpvr2srntsx\" to finish deleting: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9690275d-f0a2-4269-97f3-ff42287a50a3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9690275d-f0a2-4269-97f3-ff42287a50a3\",\n      \"name\": \"9690275d-f0a2-4269-97f3-ff42287a50a3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:08:51.225823952Z\",\n      \"endTime\": \"2026-09-16T15:32:51.218856846Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss5qjn2q2n7cj829v2pum0bhr8rsof2 still exists (deletion dispatched at 2026-09-16T15:08:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001654180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-vwpvr2srntsx\" to finish deleting: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9690275d-f0a2-4269-97f3-ff42287a50a3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9690275d-f0a2-4269-97f3-ff42287a50a3\",\n      \"name\": \"9690275d-f0a2-4269-97f3-ff42287a50a3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:08:51.225823952Z\",\n      \"endTime\": \"2026-09-16T15:32:51.218856846Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss5qjn2q2n7cj829v2pum0bhr8rsof2 still exists (deletion dispatched at 2026-09-16T15:08:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"e8bc9a8486674bdc1a81049ad931ccbb5a9f98bfa53795eca763544cc96cde0e","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"9cfc23c00b3b6ff4e037fcfa63ecc4ff6f6077b374d4b670e533df57b7d90b23","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"46b2146b5bea1fbbe7547a18b5b7e0a96d274edde0ce207e2338267f45f60742","pr_number":6976,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"50012cc84060fc59db5e840f9933bf1a542baf6598061d9d1e54da53aa955055","failure_pattern":"permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews","search_query":"permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.41208791208791207,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,2,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.086s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001018980\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/volumeattachments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiextensions.k8s.io/customresourcedefinitions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list policy/poddisruptionbudgets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list autoscaling/horizontalpodautoscalers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list coordination.k8s.io/leases: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list discovery.k8s.io/endpointslices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list certificates.k8s.io/certificatesigningrequests: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list scheduling.k8s.io/priorityclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list node.k8s.io/runtimeclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/mutatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/validatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiregistration.k8s.io/apiservices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/clusterversions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/clusteroperators: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/infrastructures: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list security.openshift.io/securitycontextconstraints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list route.openshift.io/routes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list image.openshift.io/imagestreams: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.086s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001018980\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/volumeattachments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiextensions.k8s.io/customresourcedefinitions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list policy/poddisruptionbudgets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list autoscaling/horizontalpodautoscalers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list coordination.k8s.io/leases: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list discovery.k8s.io/endpointslices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list certificates.k8s.io/certificatesigningrequests: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list scheduling.k8s.io/priorityclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list node.k8s.io/runtimeclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/mutatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/validatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiregistration.k8s.io/apiservices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/clusterversions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/clusteroperators: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/infrastructures: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list security.openshift.io/securitycontextconstraints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list route.openshift.io/routes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list image.openshift.io/imagestreams: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.100s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001341f60\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.100s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001341f60\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.093s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000ff3040\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/volumeattachments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiextensions.k8s.io/customresourcedefinitions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list policy/poddisruptionbudgets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list autoscaling/horizontalpodautoscalers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list coordination.k8s.io/leases: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list discovery.k8s.io/endpointslices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list certificates.k8s.io/certificatesigningrequests: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list scheduling.k8s.io/priorityclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list node.k8s.io/runtimeclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/mutatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/validatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.093s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000ff3040\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/volumeattachments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiextensions.k8s.io/customresourcedefinitions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list policy/poddisruptionbudgets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list autoscaling/horizontalpodautoscalers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list coordination.k8s.io/leases: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list discovery.k8s.io/endpointslices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list certificates.k8s.io/certificatesigningrequests: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list scheduling.k8s.io/priorityclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list node.k8s.io/runtimeclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/mutatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/validatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495749199433728","occurred_at":"2026-09-17T08:03:06Z","signature_id":"d145278289b57df8d05b7f53866f25f2a3e6803008a36d7c5b0f96179504086e","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100285588732121088","occurred_at":"2026-09-16T18:08:00Z","signature_id":"7f3c6a5d4502814ef937d1139cc649fca2dcf5ac7887cdb4d24e35c02e2550c3","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100188464501755904","occurred_at":"2026-09-16T11:42:04Z","signature_id":"b62e9e3d53137e1d7201b9bf6b9b273592900eb1294c925f09b1fe4ee1b51c90","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","failure_pattern":"private ingress should not be reachable from outside the VNet, but connection succeeded","search_query":"private ingress should not be reachable from outside the VNet, but connection succeeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.41208791208791207,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,2,0,0,1,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred.  Got:\n    \u003cnil\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred.  Got:\n    \u003cnil\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101401417230061568","occurred_at":"2026-09-19T20:01:54Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":7019,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100273649465430016","occurred_at":"2026-09-16T17:20:33Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252605216198656","occurred_at":"2026-09-16T15:56:55Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":6933,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wngrxcmwz6zh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-wngrxcmwz6zh, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.41208791208791207,"anchor_occurrences":3,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,0,1,2,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f1dd40\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-xrnlcrchtggh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-xrnlcrchtggh, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f1dd40\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-xrnlcrchtggh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-xrnlcrchtggh, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014fc8a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-zvg8dvqqf6zz\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zvg8dvqqf6zz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014fc8a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-zvg8dvqqf6zz\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zvg8dvqqf6zz, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009db7a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wngrxcmwz6zh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-wngrxcmwz6zh, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009db7a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wngrxcmwz6zh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-wngrxcmwz6zh, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495749199433728","occurred_at":"2026-09-17T08:03:06Z","signature_id":"d7f2fd6c375699dace4737f150cc1d0c82379eed84d0d265ff57b17f49ba73d8","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"a4ab9812dd60d9325a7127735e40ad640073cecf361f77bd3c75a60c7b17bd98","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"f366c8a9d3b487352e0d087e494e88beecbef611270fb1a6b15974fec8f76394","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.27472527472527475,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,3,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017e8150\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-5mwmp59kkrll\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\",\n      \"name\": \"0000780c-679b-479e-b61c-e08211991339\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:27:09.508692971Z\",\n      \"endTime\": \"2026-09-18T17:51:11.117212687Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti77pkq7agiit4e9p66ic3479hnrq5 still exists (deletion dispatched at 2026-09-18T17:27:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017e8150\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-5mwmp59kkrll\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\",\n      \"name\": \"0000780c-679b-479e-b61c-e08211991339\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:27:09.508692971Z\",\n      \"endTime\": \"2026-09-18T17:51:11.117212687Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti77pkq7agiit4e9p66ic3479hnrq5 still exists (deletion dispatched at 2026-09-18T17:27:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019d60f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vmz65bf72428\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n      \"name\": \"e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T07:07:58.803605122Z\",\n      \"endTime\": \"2026-09-18T07:32:03.445509106Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95msfbb4nhua5a6pq8m3044dhe0se still exists (deletion dispatched at 2026-09-18T07:08:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019d60f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vmz65bf72428\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n      \"name\": \"e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T07:07:58.803605122Z\",\n      \"endTime\": \"2026-09-18T07:32:03.445509106Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95msfbb4nhua5a6pq8m3044dhe0se still exists (deletion dispatched at 2026-09-18T07:08:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c80150\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nglkjctcmb9p\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n      \"name\": \"9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T07:10:06.620770558Z\",\n      \"endTime\": \"2026-09-18T07:34:13.53272476Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95o86ftkou4io40i8o12g13uerlpd still exists (deletion dispatched at 2026-09-18T07:10:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c80150\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nglkjctcmb9p\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n      \"name\": \"9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T07:10:06.620770558Z\",\n      \"endTime\": \"2026-09-18T07:34:13.53272476Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95o86ftkou4io40i8o12g13uerlpd still exists (deletion dispatched at 2026-09-18T07:10:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"52f64f30d917c25349ded7085a1164f0e2615c20f54359a71e907e0ab61c033a","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","signature_id":"8db752092d73d2ebfaaaef7b1150f759438bf639db4c2c7645ebdce228946dcf","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","signature_id":"cc794ab286f7ef163ad70de687149ee27cd59dcc48d24d280de823796404177d","pr_number":6969,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f7e85048bd6764cc5537fb876507c8173ed1c8737be07b8c67fd9fcfdae58024","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,3,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001580078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dfbjv854d6w5\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\",\n      \"name\": \"d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:36:31.257109777Z\",\n      \"endTime\": \"2026-09-16T10:00:32.532474531Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss1375769r7k6ubu137jlbqtq9j35it still exists (deletion dispatched at 2026-09-16T09:36:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001580078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dfbjv854d6w5\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\",\n      \"name\": \"d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:36:31.257109777Z\",\n      \"endTime\": \"2026-09-16T10:00:32.532474531Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss1375769r7k6ubu137jlbqtq9j35it still exists (deletion dispatched at 2026-09-16T09:36:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011720a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-4cfzlz\" in resourcegroup=\"rg-zstream-upgrade-5-0-2bmmvg6nm4m6\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49e79c0-b564-4703-ad68-3e8fbb555ee7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49e79c0-b564-4703-ad68-3e8fbb555ee7\",\n      \"name\": \"a49e79c0-b564-4703-ad68-3e8fbb555ee7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:52:39.028633437Z\",\n      \"endTime\": \"2026-09-16T10:16:42.563449333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss136411ov65aeqqn1ut2t4iprrujv9 still exists (deletion dispatched at 2026-09-16T09:52:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011720a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-4cfzlz\" in resourcegroup=\"rg-zstream-upgrade-5-0-2bmmvg6nm4m6\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49e79c0-b564-4703-ad68-3e8fbb555ee7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49e79c0-b564-4703-ad68-3e8fbb555ee7\",\n      \"name\": \"a49e79c0-b564-4703-ad68-3e8fbb555ee7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:52:39.028633437Z\",\n      \"endTime\": \"2026-09-16T10:16:42.563449333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss136411ov65aeqqn1ut2t4iprrujv9 still exists (deletion dispatched at 2026-09-16T09:52:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010be048\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-lq45ld6h888d\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76270801-127d-4225-97dc-06e57322bf5a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76270801-127d-4225-97dc-06e57322bf5a\",\n      \"name\": \"76270801-127d-4225-97dc-06e57322bf5a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:37:08.667361714Z\",\n      \"endTime\": \"2026-09-16T10:01:12.766789046Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss13j30utlk9cm4u8n21tm903korv38 still exists (deletion dispatched at 2026-09-16T09:37:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010be048\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-lq45ld6h888d\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76270801-127d-4225-97dc-06e57322bf5a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76270801-127d-4225-97dc-06e57322bf5a\",\n      \"name\": \"76270801-127d-4225-97dc-06e57322bf5a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:37:08.667361714Z\",\n      \"endTime\": \"2026-09-16T10:01:12.766789046Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss13j30utlk9cm4u8n21tm903korv38 still exists (deletion dispatched at 2026-09-16T09:37:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"3ce6a891cb0d07c7ce901cd0ee33f19037e602ebe5639fc8eb520176e22be57f","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"5a758cc0e5fd4d4f0a2773989c77d2a7c459fe910fa5486bed83da0d068c3f31","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"70756564bd668b5819cf49c7d07af12de78c4ca05f5acb0024a436a269a2a249","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c2687040eef152546e303fe3ce6da2f92a92db4c9a3989bfdcd2df869f68897f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.27472527472527475,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f90b40\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8r8jd5phj8ps\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\",\n      \"name\": \"de938633-4e59-468a-9378-96c48d8ef3fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T03:48:43.242131083Z\",\n      \"endTime\": \"2026-09-18T04:17:44.681797406Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 16m33s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f90b40\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8r8jd5phj8ps\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\",\n      \"name\": \"de938633-4e59-468a-9378-96c48d8ef3fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T03:48:43.242131083Z\",\n      \"endTime\": \"2026-09-18T04:17:44.681797406Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 16m33s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-lzpm5g\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d707c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-lzpm5g: failed waiting for cluster=\"cp-ystream-upgrade-4-22-lzpm5g\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-v5f9s54xdgwx\" to finish creating: GET https://rp.j9719424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cff0006a-d905-424f-9efd-d8db75f2fd05\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cff0006a-d905-424f-9efd-d8db75f2fd05\",\n      \"name\": \"cff0006a-d905-424f-9efd-d8db75f2fd05\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:49:09.853286386Z\",\n      \"endTime\": \"2026-09-16T21:18:11.008528958Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 13m4s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-lzpm5g\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d707c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-lzpm5g: failed waiting for cluster=\"cp-ystream-upgrade-4-22-lzpm5g\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-v5f9s54xdgwx\" to finish creating: GET https://rp.j9719424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cff0006a-d905-424f-9efd-d8db75f2fd05\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cff0006a-d905-424f-9efd-d8db75f2fd05\",\n      \"name\": \"cff0006a-d905-424f-9efd-d8db75f2fd05\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:49:09.853286386Z\",\n      \"endTime\": \"2026-09-16T21:18:11.008528958Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 13m4s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","signature_id":"a6058215e786bb088f449c6f95b45c11ce369a5297dbcc0542779a89a2c66add","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309343839719424","occurred_at":"2026-09-16T19:42:24Z","signature_id":"af5ff2a9735a8e441c18465437503752dd1901dae01a546fbec14adb73b0c7e5","pr_number":6403,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.27472527472527475,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,1,0,0,1],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001f2540\u003e: \n    failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-g8jlwfczq87l\" to finish updating: GET https://rp.j7871744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001f2540\u003e: \n    failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-g8jlwfczq87l\" to finish updating: GET https://rp.j7871744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--rrwf84jdrq9t/cluster-hs-pre-dlzcc5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007ce420\u003e: \n    failed to create HCP cluster cluster-hs-pre-dlzcc5: failed waiting for cluster=\"cluster-hs-pre-dlzcc5\" in resourcegroup=\"rg-hs-pre--rrwf84jdrq9t\" to finish creating: GET https://rp.j1442176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--rrwf84jdrq9t/cluster-hs-pre-dlzcc5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007ce420\u003e: \n    failed to create HCP cluster cluster-hs-pre-dlzcc5: failed waiting for cluster=\"cluster-hs-pre-dlzcc5\" in resourcegroup=\"rg-hs-pre--rrwf84jdrq9t\" to finish creating: GET https://rp.j1442176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101522358387871744","occurred_at":"2026-09-20T04:02:00Z","signature_id":"07b9cc47f1337561bab912a04c50994e47b493cb94f69944313b85e20e5e8827","pr_number":7019,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100735274731442176","occurred_at":"2026-09-17T23:54:53Z","signature_id":"a9eee6e574575c9254a566a1c15978b6cdd24af6e7e61c298d051c731edd627c","pr_number":6939,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-8ld2f474grdt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-8ld2f474grdt, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.27472527472527475,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["STG"],"trend_counts":[0,0,0,0,1,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00255de60\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-8ld2f474grdt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-8ld2f474grdt, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00255de60\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-8ld2f474grdt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-8ld2f474grdt, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.33\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017a97a0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-plrmgv\" resourcegroup=\"rg-np-version-upgrade-plrmgv-2td2q96dfsp2\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-plrmgv in resource group rg-np-version-upgrade-plrmgv-2td2q96dfsp2, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.33\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017a97a0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-plrmgv\" resourcegroup=\"rg-np-version-upgrade-plrmgv-2td2q96dfsp2\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-plrmgv in resource group rg-np-version-upgrade-plrmgv-2td2q96dfsp2, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101014543869153280","occurred_at":"2026-09-18T18:24:36Z","signature_id":"96d0afdee41d99182089b8922944bd77cbf90a6f0b4ca2fe3e238fbfdaae440e","pr_number":6639,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100487527256297472","occurred_at":"2026-09-17T07:30:18Z","signature_id":"e98964207b5fd40111448e87aa9fa8936a8a788c4325943a9b61b7b5870f98ff","pr_number":6952,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.27472527472527475,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,1,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T21:40:09Z\n  Ended:       2026-09-17T21:58:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j1728768-svc\", component=\"backend\", name=\"externalauthclusterservicecreate\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue externalauthclusterservicecreate has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T15:28:20Z\n  Ended:       2026-09-16T15:40:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j8680064-svc\", component=\"backend\", name=\"dataplaneoidcfederation\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue dataplaneoidcfederation has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100680641501728768","occurred_at":"2026-09-17T20:17:48Z","signature_id":"f3ebbdf3ff2dce8e5f0f8347ed14b1024c81d64efecc73041c83a810441bae40","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"9efd97c6c35dce815008610c2dc8e01945d6be74819a30c6714b0f4b7f9ad9da","pr_number":6983,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.27472527472527475,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":2}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-18T18:04:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1089664-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/services-j1089664\", namespace=\"klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5 on cluster ci01-j1089664-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-18T18:05:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1089664-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", lease=\"governance-policy-framework\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/services-j1089664\", namespace=\"klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease governance-policy-framework in namespace klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5 on cluster ci01-j1089664-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-18T18:05:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1089664-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", lease=\"config-policy-controller\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/services-j1089664\", namespace=\"klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease config-policy-controller in namespace klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5 on cluster ci01-j1089664-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:54:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j4807552-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.89:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4807552/providers/microsoft.monitor/accounts/services-j4807552\", namespace=\"klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-zvqnn\", prometheus=\"prometheus/prometheus\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn on cluster ci01-j4807552-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:54:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j4807552-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.89:8080\", job=\"kube-state-metrics\", lease=\"config-policy-controller\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4807552/providers/microsoft.monitor/accounts/services-j4807552\", namespace=\"klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-zvqnn\", prometheus=\"prometheus/prometheus\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease config-policy-controller in namespace klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn on cluster ci01-j4807552-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:54:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j4807552-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.89:8080\", job=\"kube-state-metrics\", lease=\"governance-policy-framework\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4807552/providers/microsoft.monitor/accounts/services-j4807552\", namespace=\"klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-zvqnn\", prometheus=\"prometheus/prometheus\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease governance-policy-framework in namespace klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn on cluster ci01-j4807552-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"b2d9d84ce6e1c74ea14279f086b91ae4fc06d24e40b82a7ad2aa54e78ffa7d4b","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"ed2b24c463c24323e023a9ff56bf6535b99fca0fd95a6e6a5f3b0c1f68b41036","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.27472527472527475,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,2,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0014180c0\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-phbtk2 namespace are ready: failed to list pods in cilium-connectivity-check-phbtk2 namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0014180c0\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-phbtk2 namespace are ready: failed to list pods in cilium-connectivity-check-phbtk2 namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00168f0c8\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-txdwxr namespace are ready: failed to list pods in cilium-connectivity-check-txdwxr namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00168f0c8\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-txdwxr namespace are ready: failed to list pods in cilium-connectivity-check-txdwxr namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100434265043898368","occurred_at":"2026-09-17T03:58:47Z","signature_id":"297e17fd0f8b231f49a6a64a53967d8a3faf0348a5fdc0a0860997681f536656","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"2bd939aaad187ab78804671b6970c3fb300c0b1bc1be22955c06bc837de68bb9","pr_number":6976,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.27472527472527475,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","occurrences":2}],"full_error_samples":["time=2026-09-16T10:35:02.273Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T10:35:02.716Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6352640-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6352640-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:ad132446-7529-417f-aff8-31d826ffb6f2 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#ad132446-7529-417f-aff8-31d826ffb6f2 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T10:35:02.716Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:35:02.863Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T10:35:03.042Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T10:35:03.042Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:35:03.143Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:45:05.949Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T10:45:05.949Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T10:46:05.955Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T10:46:06.637Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6352640-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6352640-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:ad132446-7529-417f-aff8-31d826ffb6f2 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#ad132446-7529-417f-aff8-31d826ffb6f2 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T10:46:06.637Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:07.013Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T10:46:07.408Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:07.408Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:07.670Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-16T10:46:08.488Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:08.489Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:08.888Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:09.401Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T10:46:09.747Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T10:46:09.959Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:10.361Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:10.729Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:11.065Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T10:46:11.065Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:11.148Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-16T10:56:12.973Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T10:56:12.973Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T10:57:12.977Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T10:57:13.486Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6352640-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6352640-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:ad132446-7529-417f-aff8-31d826ffb6f2 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#ad132446-7529-417f-aff8-31d826ffb6f2 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T10:57:13.486Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:13.807Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T10:57:14.174Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:14.174Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:14.404Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-16T10:57:15.329Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:15.329Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:15.688Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:15.987Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T10:57:16.345Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T10:57:16.535Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:16.747Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:16.984Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:17.208Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T10:57:17.208Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:17.300Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-16T11:05:02.358Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-16T11:05:02.359Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","time=2026-09-16T08:43:27.720Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T08:43:28.143Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6406656-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6406656-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:510fcd49-73a6-4643-b104-d1da046c7a87 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#510fcd49-73a6-4643-b104-d1da046c7a87 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T08:43:28.143Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:43:28.283Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T08:43:28.434Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T08:43:28.434Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:43:28.493Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:53:31.202Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T08:53:31.203Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T08:54:31.204Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T08:54:31.661Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6406656-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6406656-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:510fcd49-73a6-4643-b104-d1da046c7a87 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#510fcd49-73a6-4643-b104-d1da046c7a87 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T08:54:31.661Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:31.919Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T08:54:32.413Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:32.413Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:32.686Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-16T08:54:33.497Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:33.498Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:33.902Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:34.404Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T08:54:34.757Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T08:54:34.953Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:35.182Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:35.544Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:36.068Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T08:54:36.068Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:36.176Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-16T09:04:37.727Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T09:04:37.727Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T09:05:37.733Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T09:05:38.373Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6406656-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6406656-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:510fcd49-73a6-4643-b104-d1da046c7a87 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#510fcd49-73a6-4643-b104-d1da046c7a87 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T09:05:38.373Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:38.676Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T09:05:39.023Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:39.023Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:39.181Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-16T09:05:40.018Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:40.018Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:40.264Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:40.639Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T09:05:41.058Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T09:05:41.352Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:41.683Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:42.063Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:42.276Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T09:05:42.276Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:42.359Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-16T09:13:27.645Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-16T09:13:27.646Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161854826352640","occurred_at":"2026-09-16T09:56:19Z","signature_id":"64a318e95cdcfb12234888d476e960be3b284497df057c04c923477d134c5a2a","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132066686406656","occurred_at":"2026-09-16T07:57:56Z","signature_id":"d08b0cbd15edc9cffd195b1bd96feff56d9f329451cf7393af1293efbc3c0c96","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-r67qjj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000814820\u003e: \n    DNS for route host agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-r67qjj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000814820\u003e: \n    DNS for route host agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-z9mjpt\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010173e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-z9mjpt\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010173e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252605216198656","occurred_at":"2026-09-16T15:56:55Z","signature_id":"662575cd71f8c3606ecf799e2d76b50f1b3ecb8c083205f994e08e750a8a40a2","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252605216198656","occurred_at":"2026-09-16T15:56:55Z","signature_id":"d9283aaa1d82489c4a0f2f12dd912bceba77f21d6bae60f6a616ab523d7ace51","pr_number":6933,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"72716b475dfdb36ca6de5b4a7866bbc958e1921fb2c6ea0b3467c231190cd12c","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f86780\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-wg24nd-tjd5hx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-wg24nd-tjd5hx\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: DeploymentNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentNotFound\",\n        \"message\": \"Deployment 'mi-np-version-upgrade-cluster-wg24nd-tjd5hx' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f86780\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-wg24nd-tjd5hx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-wg24nd-tjd5hx\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: DeploymentNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentNotFound\",\n        \"message\": \"Deployment 'mi-np-version-upgrade-cluster-wg24nd-tjd5hx' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:104]: failed to create customer resources for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a8d220\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-cc6rq5\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-cc6rq5\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: DeploymentNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentNotFound\",\n        \"message\": \"Deployment 'mi-pullsecret-hcp-cluster-cc6rq5' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:104]: failed to create customer resources for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a8d220\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-cc6rq5\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-cc6rq5\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: DeploymentNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentNotFound\",\n        \"message\": \"Deployment 'mi-pullsecret-hcp-cluster-cc6rq5' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","occurred_at":"2026-09-20T09:53:04Z","signature_id":"7315612725ec25e205b11f3adbd85547f42e9e5cff4ae5b4c955333bbd4ccd4f","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","occurred_at":"2026-09-20T09:53:04Z","signature_id":"8a93b4c61b0436ed275798352e60b97494e315997f9735dcbbf3fcea1cb8b1bc","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1d42c7dad9427d57e1b19754ffe767a1fc187725c1c8da2f03ad1f6733ce9b5d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aaaa60\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6mp8vkxcbh5s\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d35dfb81-23c2-4a18-80a3-7d5bf5893728\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d35dfb81-23c2-4a18-80a3-7d5bf5893728\",\n      \"name\": \"d35dfb81-23c2-4a18-80a3-7d5bf5893728\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T10:51:43.295429479Z\",\n      \"endTime\": \"2026-09-16T11:20:49.710317459Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aaaa60\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6mp8vkxcbh5s\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d35dfb81-23c2-4a18-80a3-7d5bf5893728\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d35dfb81-23c2-4a18-80a3-7d5bf5893728\",\n      \"name\": \"d35dfb81-23c2-4a18-80a3-7d5bf5893728\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T10:51:43.295429479Z\",\n      \"endTime\": \"2026-09-16T11:20:49.710317459Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011044c0\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-95l622wn4sm9\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b24629a-f978-4a3d-9d8d-8d5b6776590f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b24629a-f978-4a3d-9d8d-8d5b6776590f\",\n      \"name\": \"1b24629a-f978-4a3d-9d8d-8d5b6776590f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T10:52:02.869364535Z\",\n      \"endTime\": \"2026-09-16T11:21:09.719035401Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011044c0\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-95l622wn4sm9\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b24629a-f978-4a3d-9d8d-8d5b6776590f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b24629a-f978-4a3d-9d8d-8d5b6776590f\",\n      \"name\": \"1b24629a-f978-4a3d-9d8d-8d5b6776590f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T10:52:02.869364535Z\",\n      \"endTime\": \"2026-09-16T11:21:09.719035401Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"7299032a0750fba48148c45abb91daa5c3ae005efe59b617e7dc255e091f7a66","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"84f288aa90f1963969fa6df3b97105ac35a5dcb0b0f4f283ef7675cb82451eeb","pr_number":6791,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"44fc5c170ee5969da686dbaeb61a39c91b4e9f01cf7a4324b16151d7f8a7811a","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005c0f60\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2fppphk96nck\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/648b059b-8450-4d5d-bb2f-41ba8931b7e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/648b059b-8450-4d5d-bb2f-41ba8931b7e5\",\n      \"name\": \"648b059b-8450-4d5d-bb2f-41ba8931b7e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-20T17:50:38.502314717Z\",\n      \"endTime\": \"2026-09-20T18:19:40.439163186Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005c0f60\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2fppphk96nck\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/648b059b-8450-4d5d-bb2f-41ba8931b7e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/648b059b-8450-4d5d-bb2f-41ba8931b7e5\",\n      \"name\": \"648b059b-8450-4d5d-bb2f-41ba8931b7e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-20T17:50:38.502314717Z\",\n      \"endTime\": \"2026-09-20T18:19:40.439163186Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000696760\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-lzkpz5ftlkll\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83245618-9845-4872-97a3-4ee524027d39\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83245618-9845-4872-97a3-4ee524027d39\",\n      \"name\": \"83245618-9845-4872-97a3-4ee524027d39\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-20T17:50:02.101717721Z\",\n      \"endTime\": \"2026-09-20T18:19:10.391207338Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000696760\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-lzkpz5ftlkll\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83245618-9845-4872-97a3-4ee524027d39\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83245618-9845-4872-97a3-4ee524027d39\",\n      \"name\": \"83245618-9845-4872-97a3-4ee524027d39\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-20T17:50:02.101717721Z\",\n      \"endTime\": \"2026-09-20T18:19:10.391207338Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"e6538054ab0559cddb2c0ed63235c48a4d5cd57bef25e042c4c66fb7e93ba92e","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"f37d1bd86a2dec4c46be9ce8fb4c03ed08425d29a0d32e20dd962b464a680228","pr_number":6965,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"3ec8dfede2b7d12b6200fcfa62f782082fd9d3f96c215ae2c1de00861966d250","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":2,"after_last_push_count":0,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,2,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d7ccc0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xp4rlvdpbgj6\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\",\n      \"name\": \"b344e921-2b47-4118-aac7-81c45cb67cc0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T06:13:19.863238771Z\",\n      \"endTime\": \"2026-09-18T06:42:22.545828006Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d7ccc0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xp4rlvdpbgj6\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\",\n      \"name\": \"b344e921-2b47-4118-aac7-81c45cb67cc0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T06:13:19.863238771Z\",\n      \"endTime\": \"2026-09-18T06:42:22.545828006Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a46a00\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jt9b8z9tj78s\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n      \"name\": \"16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T06:13:10.341418654Z\",\n      \"endTime\": \"2026-09-18T06:42:12.545110519Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a46a00\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jt9b8z9tj78s\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n      \"name\": \"16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T06:13:10.341418654Z\",\n      \"endTime\": \"2026-09-18T06:42:12.545110519Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","signature_id":"0d8835203476a034717107aadec991b28272c94910dad48bde3bb462bb56ac9b","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","signature_id":"4fb56cfa0c5d81c59f21dba90bd2b4aeaa2b18f5202205163a7a19b83e403f4e","pr_number":6157,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1","failure_pattern":"expected node pool nodes to report at least one release image ref before upgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:474]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:474]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","pr_number":7032,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"240d7e65fa5e2b6ff5542fc888978c21e203da5524e2e297822471f59f51aa10","failure_pattern":"\"error\": {","search_query":"\"error\": {","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:139]: failed to get backup schedules for cadence check\nUnexpected error:\n    \u003c*errors.errorString | 0xc0010c3e40\u003e: \n    expected status 200, got 412: {\n        \"error\": {\n            \"code\": \"InvalidResource\",\n            \"message\": \"kube-applier client not available for management cluster /providers/microsoft.redhatopenshift/stamps/1/managementclusters/default\"\n        }\n    }\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:139]: failed to get backup schedules for cadence check\nUnexpected error:\n    \u003c*errors.errorString | 0xc0010c3e40\u003e: \n    expected status 200, got 412: {\n        \"error\": {\n            \"code\": \"InvalidResource\",\n            \"message\": \"kube-applier client not available for management cluster /providers/microsoft.redhatopenshift/stamps/1/managementclusters/default\"\n        }\n    }\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","signature_id":"06ba3d2a9582d2a18869bdd5ae1df4321a4d0977a7be4bdd0e3cc1af2ec0608a","pr_number":6157,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-update-nodes: failed waiting for nodepool=\"np-update-nodes\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-ckch4xg6nxh4\" to finish creating: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e55eeddb-0ead-432e-a110-c696080abf84\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e55eeddb-0ead-432e-a110-c696080abf84\",\n  \"name\": \"e55eeddb-0ead-432e-a110-c696080abf84\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-16T20:39:13.615216045Z\",\n  \"endTime\": \"2026-09-16T20:58:16.977517435Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"node pool creation did not complete before the deadline\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-update-nodes: failed waiting for nodepool=\"np-update-nodes\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-ckch4xg6nxh4\" to finish creating: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e55eeddb-0ead-432e-a110-c696080abf84\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e55eeddb-0ead-432e-a110-c696080abf84\",\n  \"name\": \"e55eeddb-0ead-432e-a110-c696080abf84\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-16T20:39:13.615216045Z\",\n  \"endTime\": \"2026-09-16T20:58:16.977517435Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"node pool creation did not complete before the deadline\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"663948fad2f655a8f86778686b2675b46d2a717e3dc1afdecbbc520028b3d155","pr_number":6945,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f59e236e067df212a87e97ef00e5cb14f86f6db93d160e4c5cd41734c06ea1b3","failure_pattern":"Admin credentials did not return 409 conflict error while cluster is deploying","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:144]: Admin credentials did not return 409 conflict error while cluster is deploying","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:144]: Admin credentials did not return 409 conflict error while cluster is deploying\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:144]: Admin credentials did not return 409 conflict error while cluster is deploying"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100237146764873728","occurred_at":"2026-09-16T14:55:30Z","signature_id":"6628b29b597aa4c3bb50125f87195e6a58208831ab9330e6078d3d1c523d7af7","pr_number":6752,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4f70c086dc5495ea728fc39aab23a54b6cce4072ba8ccfc36ee6980300ba5299","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-j7tx89s2rwk5\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-j7tx89s2rwk5, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:276]: failed to create external auth config \"ext-auth-cidr\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002990240\u003e: \n    failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-j7tx89s2rwk5\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-j7tx89s2rwk5, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:276]: failed to create external auth config \"ext-auth-cidr\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002990240\u003e: \n    failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-j7tx89s2rwk5\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-j7tx89s2rwk5, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100680641501728768","occurred_at":"2026-09-17T20:17:48Z","signature_id":"bf84ee4449e687d0ad4cd4859ef7515e9901f5ded80364d34ebb748aaa5b41df","pr_number":6999,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a16108\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a16108\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"e37087f31e7094fe45c7ae2868faf05fde32666577b47902fe94fbe270287c3b","pr_number":6993,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fa65c8375667d2f86c4c029a1dd32730b8452c4c21edd3768c8aaedc846a9603","failure_pattern":"ERROR CODE: Canceled; detail message This operation was superseded by another; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Canceled","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-l9txx6 with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d2ad80\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-l9txx6: failed waiting for cluster=\"np-version-upgrade-cluster-l9txx6\" in resourcegroup=\"rg-np-version-upgrade-l9txx6-mwmzt2xrcxvt\" to finish creating: GET https://rp.j2215680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ca9f3ef6-29fc-49b7-a060-580e48d94813\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: Canceled\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ca9f3ef6-29fc-49b7-a060-580e48d94813\",\n      \"name\": \"ca9f3ef6-29fc-49b7-a060-580e48d94813\",\n      \"status\": \"Canceled\",\n      \"startTime\": \"2026-09-19T18:21:32.692382703Z\",\n      \"endTime\": \"2026-09-19T18:21:36.063277066Z\",\n      \"error\": {\n        \"code\": \"Canceled\",\n        \"message\": \"This operation was superseded by another\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-l9txx6 with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d2ad80\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-l9txx6: failed waiting for cluster=\"np-version-upgrade-cluster-l9txx6\" in resourcegroup=\"rg-np-version-upgrade-l9txx6-mwmzt2xrcxvt\" to finish creating: GET https://rp.j2215680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ca9f3ef6-29fc-49b7-a060-580e48d94813\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: Canceled\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ca9f3ef6-29fc-49b7-a060-580e48d94813\",\n      \"name\": \"ca9f3ef6-29fc-49b7-a060-580e48d94813\",\n      \"status\": \"Canceled\",\n      \"startTime\": \"2026-09-19T18:21:32.692382703Z\",\n      \"endTime\": \"2026-09-19T18:21:36.063277066Z\",\n      \"error\": {\n        \"code\": \"Canceled\",\n        \"message\": \"This operation was superseded by another\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7024/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101365667302215680","occurred_at":"2026-09-19T17:39:51Z","signature_id":"31abd0b67e969606a65d92c1ada057ebfc9c0d1692c93463a992c05e6a275c62","pr_number":7024,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1019af4c9c47d1aa706702f78998f006a0e49b021ad920d44d356df2637fa18e","failure_pattern":"ERROR CODE: Conflict; detail message Cannot update resource while resource is \"accepted\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-l9txx6 with version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00167c880\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-l9txx6: failed starting cluster creation \"np-version-upgrade-cluster-l9txx6\" in resourcegroup=\"rg-np-version-upgrade-l9txx6-mwmzt2xrcxvt\": PUT https://rp.j2215680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-l9txx6-mwmzt2xrcxvt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-l9txx6\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"accepted\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-l9txx6-mwmzt2xrcxvt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-l9txx6\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-l9txx6 with version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00167c880\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-l9txx6: failed starting cluster creation \"np-version-upgrade-cluster-l9txx6\" in resourcegroup=\"rg-np-version-upgrade-l9txx6-mwmzt2xrcxvt\": PUT https://rp.j2215680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-l9txx6-mwmzt2xrcxvt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-l9txx6\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"accepted\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-l9txx6-mwmzt2xrcxvt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-l9txx6\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7024/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101365667302215680","occurred_at":"2026-09-19T17:39:51Z","signature_id":"dfa687b8ebe45393643ad662164c245b2ff000e0e3a18f9d0335eaa169ceedd1","pr_number":7024,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"377edc68350e3289212c2970b00ba8365c9d949a03379484101a64858ed9715c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message The collection with name [Resources] in provided scope [/dbs/\u003ccosmos-account\u003e/colls/Resources] could not be found.\\\\\\ \\\\ ActivityId: \u003cuuid\u003e, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0; provider Microsoft.DocumentDB","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1}],"full_error_samples":["time=2026-09-16T13:31:49.803Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-09-16T13:31:51.360Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-09-16T13:31:52.622Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=6a654e03e0d8cd3ba2dcf4b7ca7ea7b9161da4e2734cccce092b3f0e303e2fe9 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6373376-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6a654e03e0d8cd3ba2dcf4b7ca7ea7b9161da4e2734cccce092b3f0e303e2fe9\ntime=2026-09-16T13:40:24.258Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6373376-svc/providers/Microsoft.Resources/deployments/6a654e03e0d8cd3ba2dcf4b7ca7ea7b9161da4e2734cccce092b3f0e303e2fe9/operationStatuses/08584120413735528505\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6373376/providers/Microsoft.Resources/deployments/rp_cosmos_db\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6373376/providers/Microsoft.DocumentDB/databaseAccounts/arohcpci01-rp-j6373376/sqlRoleAssignments/3a943d25-7e89-52ef-bdd2-d3814561859c\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The collection with name [Resources] in provided scope [/dbs/arohcpci01-rp-j6373376/colls/Resources] could not be found.\\\\\\\\r\\\\\\\\nActivityId: 385582e2-ce0e-4904-a9f2-7469a3089fa3, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100209889606373376","occurred_at":"2026-09-16T13:07:12Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6886,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"54082ad8881c5131bff996f7153979c81faa7fa74e3b195d969ebc6eb63a2c19","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/cluster-\u003cid\u003e' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '\u003ctimestamp\u003e' with correlationId '\u003cuuid\u003e', and will expire at '\u003ctimestamp\u003e' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-09-17T08:52:09.303Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T08:52:10.143Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-17T08:52:11.020Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=e5c452463d5a2ca118d246d0e10c4f76fe32f1478279da2b4450028ca1d7e3f8 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2728320-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fe5c452463d5a2ca118d246d0e10c4f76fe32f1478279da2b4450028ca1d7e3f8\ntime=2026-09-17T08:54:11.408Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2728320-mgmt-1/providers/Microsoft.Resources/deployments/e5c452463d5a2ca118d246d0e10c4f76fe32f1478279da2b4450028ca1d7e3f8/operationStatuses/08584119717548817375\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentActive\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2728320-mgmt-1/providers/Microsoft.Resources/deployments/cluster-34wn2l5qqmupi' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '9/17/2026 8:52:28 AM' with correlationId 'd7f46ae1-ae89-4906-81b7-ab9a15265766', and will expire at '9/24/2026 8:52:28 AM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100500788332728320","occurred_at":"2026-09-17T08:23:07Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0008473d183c6e76249427529630789e6d4f9e8f1d5bdbea4b6bd57ae83c8f06","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/svc-kv-xehipxbjxawki' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '\u003ctimestamp\u003e' with correlationId '\u003cuuid\u003e', and will expire at '\u003ctimestamp\u003e' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra-output","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra-output","occurrences":1}],"full_error_samples":["time=2026-09-16T08:27:04.451Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output description=\"Step infra-output\\n  Kind: ARM\\n  Template: templates/svc-infra-lookup.bicep\\n  Parameters: configurations/svc-infra-lookup.tmpl.bicepparam\"\ntime=2026-09-16T08:27:04.849Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output\ntime=2026-09-16T08:27:05.446Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output deployment=eaaad5ae26368062d8da5183fba1f1cf527c2665c3e53af22a660fa55c8dc157 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7918592-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Feaaad5ae26368062d8da5183fba1f1cf527c2665c3e53af22a660fa55c8dc157\ntime=2026-09-16T08:27:35.588Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7918592-svc/providers/Microsoft.Resources/deployments/eaaad5ae26368062d8da5183fba1f1cf527c2665c3e53af22a660fa55c8dc157/operationStatuses/08584120596603628303\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentActive\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/svc-kv-xehipxbjxawki' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '9/16/2026 8:27:06 AM' with correlationId '032c9251-9fc6-4ff2-b8e3-30d15533261b', and will expire at '9/23/2026 8:27:06 AM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132012407918592","occurred_at":"2026-09-16T07:57:44Z","signature_id":"09b1458b46da16205cd209ec402488a728d06d1e258b36845618ba3741f96692","pr_number":6639,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"8aa573d68ede374d56fcd25dace5feffa8644835d49bf0c7ba52006e6de9139d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentNotFound; detail message Deployment 'svc-kv-xehipxbjxawki' could not be found.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra-output","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra-output","occurrences":1}],"full_error_samples":["time=2026-09-18T18:49:43.745Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output description=\"Step infra-output\\n  Kind: ARM\\n  Template: templates/svc-infra-lookup.bicep\\n  Parameters: configurations/svc-infra-lookup.tmpl.bicepparam\"\ntime=2026-09-18T18:49:44.412Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output\ntime=2026-09-18T18:49:45.666Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output deployment=299b33c33a8d1927824771d1411d9cc2911c3c4239fa63e4af7c6c883950aabd portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6366720-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F299b33c33a8d1927824771d1411d9cc2911c3c4239fa63e4af7c6c883950aabd\ntime=2026-09-18T18:50:16.326Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6366720-svc/providers/Microsoft.Resources/deployments/299b33c33a8d1927824771d1411d9cc2911c3c4239fa63e4af7c6c883950aabd/operationStatuses/08584118495004368821\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"NotFound\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentNotFound\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Deployment 'svc-kv-xehipxbjxawki' could not be found.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7017/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101014245486366720","occurred_at":"2026-09-18T18:23:12Z","signature_id":"09b1458b46da16205cd209ec402488a728d06d1e258b36845618ba3741f96692","pr_number":7017,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ef6114015f72fab4d7dec149451d78cb6b761172f511b8fdedd7928043056bed","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ErrCode_InsufficientVCPUQuota; detail message Insufficient vcpu quota requested \u003ccount\u003e, remaining \u003ccount\u003e for family StandardEdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: \u003curl\u003e Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-16T15:02:01.563Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-16T15:02:02.231Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-16T15:02:03.235Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=9b458a858b62ced8b861c50221db722ca7b84af5c0247e7866fc1bf068dbe8ac portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7202944-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9b458a858b62ced8b861c50221db722ca7b84af5c0247e7866fc1bf068dbe8ac\ntime=2026-09-16T15:11:05.045Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.Resources/deployments/9b458a858b62ced8b861c50221db722ca7b84af5c0247e7866fc1bf068dbe8ac/operationStatuses/08584120359627896142\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.Resources/deployments/cluster-oqca7h7ulqwlw\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7202944-mgmt-2/agentPools/userswft3\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 96, remaining 48 for family StandardEdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n                  },\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7202944-mgmt-2/agentPools/userswft2\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 96, remaining 48 for family StandardEdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n                  },\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7202944-mgmt-2/agentPools/userswft1\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 96, remaining 48 for family StandardEdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100233505077202944","occurred_at":"2026-09-16T14:41:02Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6980,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e07cda02c84cd6ea2552fbcc5179ebfd519e66665b62547f1ea88bc8136bc936","failure_pattern":"ERROR CODE: DeploymentFailed; detail code NotFound; detail message Resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/prometheus' was not found.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-20T17:31:48.163Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-20T17:31:48.670Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-20T17:31:49.380Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=113c13d4bee9395d4112e94ba8bdfafc6cef483f6067a14e96151dee740150d6 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1872640-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F113c13d4bee9395d4112e94ba8bdfafc6cef483f6067a14e96151dee740150d6\ntime=2026-09-20T17:33:49.754Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1872640-mgmt-2/providers/Microsoft.Resources/deployments/113c13d4bee9395d4112e94ba8bdfafc6cef483f6067a14e96151dee740150d6/operationStatuses/08584116813764412291\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1872640-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-post\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1872640-mgmt-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/prometheus' was not found.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"NotFound\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1872640-mgmt-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/prometheus' was not found.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101718617941872640","occurred_at":"2026-09-20T17:02:21Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"255883e0e1738fd02f2daa41d490a5ba122bcfd657ae1579f6fbfb04d4dff13d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-09-19T19:26:02.513Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=1 description=\"Step infra\\n  Kind: ARM\\n  Template: templates/mgmt-infra.bicep\\n  Parameters: configurations/mgmt-infra.tmpl.bicepparam\"\ntime=2026-09-19T19:26:03.032Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=1\ntime=2026-09-19T19:26:04.005Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=1 deployment=6b08ae6c0b84b662e34009b99333555395423c7dc45a028e443e73278d202759 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3811200-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6b08ae6c0b84b662e34009b99333555395423c7dc45a028e443e73278d202759\ntime=2026-09-19T19:28:34.855Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3811200-mgmt-1/providers/Microsoft.Resources/deployments/6b08ae6c0b84b662e34009b99333555395423c7dc45a028e443e73278d202759/operationStatuses/08584117609220848049\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3811200-mgmt-1/providers/Microsoft.Resources/deployments/ah-ci01-me-j3811200-1\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"VaultAlreadyExists\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The vault name 'ah-ci01-me-j3811200-1' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7021/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101386126613811200","occurred_at":"2026-09-19T19:00:57Z","signature_id":"6746fdba3f8e1b37727a9ea366fc436109e899f94f0c4bf2737f7d428ff3f2ba","pr_number":7021,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"fe24c8b4b28d3d2b1937803a92704c44ec574f3d1c3b740ec6d97282a1e18d39","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00230a300\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ql5j7w6fx6vn\" to finish creating: GET https://rp.j6364288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5fde05ba-68c2-437a-84ea-82765b356728\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5fde05ba-68c2-437a-84ea-82765b356728\",\n      \"name\": \"5fde05ba-68c2-437a-84ea-82765b356728\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T11:04:09.351110934Z\",\n      \"endTime\": \"2026-09-16T11:04:15.302518412Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00230a300\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ql5j7w6fx6vn\" to finish creating: GET https://rp.j6364288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5fde05ba-68c2-437a-84ea-82765b356728\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5fde05ba-68c2-437a-84ea-82765b356728\",\n      \"name\": \"5fde05ba-68c2-437a-84ea-82765b356728\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T11:04:09.351110934Z\",\n      \"endTime\": \"2026-09-16T11:04:15.302518412Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100164075236364288","occurred_at":"2026-09-16T10:05:09Z","signature_id":"59512808dbcb20b06030c9fde07924554878fee722f13b2a06c73ba1d94bd2f9","pr_number":6688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6ed58550ec6b76673ce00588c8bebee5ecb6ea4c774ba1ceae5e31e8e80cd829","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000edc320\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-d77hgbbbw6rf\" to finish creating: GET https://rp.j6676096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74e66356-25b3-4d6f-95af-f1292008b6fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74e66356-25b3-4d6f-95af-f1292008b6fd\",\n      \"name\": \"74e66356-25b3-4d6f-95af-f1292008b6fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:47:22.69341748Z\",\n      \"endTime\": \"2026-09-16T21:16:24.116227211Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 6m20s ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000edc320\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-d77hgbbbw6rf\" to finish creating: GET https://rp.j6676096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74e66356-25b3-4d6f-95af-f1292008b6fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74e66356-25b3-4d6f-95af-f1292008b6fd\",\n      \"name\": \"74e66356-25b3-4d6f-95af-f1292008b6fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:47:22.69341748Z\",\n      \"endTime\": \"2026-09-16T21:16:24.116227211Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 6m20s ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100314659356676096","occurred_at":"2026-09-16T20:03:31Z","signature_id":"ec03acc423d7d391cd0db46f80f6e282dd4d144bd87644aded93eda8c1bf31c4","pr_number":6969,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"46425ffc9312d96c5106d1b27946c7775e4ddf6d0471e792a3aaf8c2bc4d3a86","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbc120\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-w5shsm9ng8kc\" to finish creating: GET https://rp.j5021696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\",\n      \"name\": \"eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T04:52:38.625826904Z\",\n      \"endTime\": \"2026-09-17T05:21:45.494759028Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 2m17s ago; version 4.22.14 is Partial (want Completed), started 5m22s ago; version 4.22.14 is Partial (want Completed), started 12m16s ago; version 4.22.14 is Partial (want Completed), started 15m20s ago; version 4.22.14 is Partial (want Completed), started 17m23s ago; version 4.22.14 is Partial (want Completed), started 17m58s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbc120\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-w5shsm9ng8kc\" to finish creating: GET https://rp.j5021696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\",\n      \"name\": \"eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T04:52:38.625826904Z\",\n      \"endTime\": \"2026-09-17T05:21:45.494759028Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 2m17s ago; version 4.22.14 is Partial (want Completed), started 5m22s ago; version 4.22.14 is Partial (want Completed), started 12m16s ago; version 4.22.14 is Partial (want Completed), started 15m20s ago; version 4.22.14 is Partial (want Completed), started 17m23s ago; version 4.22.14 is Partial (want Completed), started 17m58s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100438339525021696","occurred_at":"2026-09-17T04:14:58Z","signature_id":"421f5b6d838c121843fe235c14d4d70af34427ca212e0849c68ed8aa29654d6b","pr_number":6986,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"5d59b4e4feb4fd290a5c1b398014539407c812b82baa8f933608971503b0bc0d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8a120\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-8prbhvsvlkq9\" to finish creating: GET https://rp.j9922816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b148abb4-8011-43a5-acf9-7d2f1f124b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b148abb4-8011-43a5-acf9-7d2f1f124b53\",\n      \"name\": \"b148abb4-8011-43a5-acf9-7d2f1f124b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:56:50.517658972Z\",\n      \"endTime\": \"2026-09-16T21:25:54.975318736Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8a120\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-8prbhvsvlkq9\" to finish creating: GET https://rp.j9922816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b148abb4-8011-43a5-acf9-7d2f1f124b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b148abb4-8011-43a5-acf9-7d2f1f124b53\",\n      \"name\": \"b148abb4-8011-43a5-acf9-7d2f1f124b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:56:50.517658972Z\",\n      \"endTime\": \"2026-09-16T21:25:54.975318736Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309676229922816","occurred_at":"2026-09-16T19:43:43Z","signature_id":"a48d7255fe4f2e537c6260d9fc791f5067ce6859f69937f424d6c26f5bc86ca9","pr_number":6894,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"676e68f8a68616ac8c57dc0fe075f4a0d00c211dff572a92461d02984b57a771","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098aa40\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-6th5jd8nbg5s\" to finish creating: GET https://rp.j2424320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd1a9666-9126-4a24-9b9a-2799ae9c069b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd1a9666-9126-4a24-9b9a-2799ae9c069b\",\n      \"name\": \"bd1a9666-9126-4a24-9b9a-2799ae9c069b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T13:57:52.148119457Z\",\n      \"endTime\": \"2026-09-16T14:26:53.36802381Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, community-operators-catalog, hosted-cluster-config-operator, catalog-operator, redhat-operators-catalog, ingress-operator, cluster-image-registry-operator, openshift-route-controller-manager, cluster-node-tuning-operator, dns-operator, redhat-marketplace-catalog, certified-operators-catalog, cluster-network-operator, packageserver, openshift-controller-manager, cluster-policy-controller, cluster-version-operator, cluster-storage-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098aa40\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-6th5jd8nbg5s\" to finish creating: GET https://rp.j2424320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd1a9666-9126-4a24-9b9a-2799ae9c069b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd1a9666-9126-4a24-9b9a-2799ae9c069b\",\n      \"name\": \"bd1a9666-9126-4a24-9b9a-2799ae9c069b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T13:57:52.148119457Z\",\n      \"endTime\": \"2026-09-16T14:26:53.36802381Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, community-operators-catalog, hosted-cluster-config-operator, catalog-operator, redhat-operators-catalog, ingress-operator, cluster-image-registry-operator, openshift-route-controller-manager, cluster-node-tuning-operator, dns-operator, redhat-marketplace-catalog, certified-operators-catalog, cluster-network-operator, packageserver, openshift-controller-manager, cluster-policy-controller, cluster-version-operator, cluster-storage-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100212360202424320","occurred_at":"2026-09-16T13:16:57Z","signature_id":"fec53f2216f9f1f9da9e1788f3642b62d62882fc08fe207f753fbff13e9390bf","pr_number":6894,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0004ec1e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-hdsrgp87vhp6\" to finish deleting: GET https://rp.j6073984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/140fc1a6-e41e-4f25-9908-25d174a04842\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/140fc1a6-e41e-4f25-9908-25d174a04842\",\n      \"name\": \"140fc1a6-e41e-4f25-9908-25d174a04842\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T21:51:08.271574522Z\",\n      \"endTime\": \"2026-09-17T22:15:12.38485138Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st0r3e7gsp00p1v6je2veqpisj6tv8b still exists (deletion dispatched at 2026-09-17T21:51:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0004ec1e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-hdsrgp87vhp6\" to finish deleting: GET https://rp.j6073984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/140fc1a6-e41e-4f25-9908-25d174a04842\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/140fc1a6-e41e-4f25-9908-25d174a04842\",\n      \"name\": \"140fc1a6-e41e-4f25-9908-25d174a04842\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T21:51:08.271574522Z\",\n      \"endTime\": \"2026-09-17T22:15:12.38485138Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st0r3e7gsp00p1v6je2veqpisj6tv8b still exists (deletion dispatched at 2026-09-17T21:51:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685654236073984","occurred_at":"2026-09-17T20:37:43Z","signature_id":"fa245128857c23599fcf8f3484b06ab6ca5868cca2b19c36072deb31d3760300","pr_number":7000,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"82fcc2cca51beebf00c7302e8f8dc3563b7995a4550dbcf53d6d84d1820ed342","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00101a2b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-fth2ns8rql9z\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8997c6b-6d2a-4f02-afe2-3927067cda87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8997c6b-6d2a-4f02-afe2-3927067cda87\",\n      \"name\": \"e8997c6b-6d2a-4f02-afe2-3927067cda87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T09:27:47.803109699Z\",\n      \"endTime\": \"2026-09-17T09:51:54.618583456Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sslv6ma6mn5k5mt36luuom09289pq58 still exists (deletion dispatched at 2026-09-17T09:27:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00101a2b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-fth2ns8rql9z\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8997c6b-6d2a-4f02-afe2-3927067cda87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8997c6b-6d2a-4f02-afe2-3927067cda87\",\n      \"name\": \"e8997c6b-6d2a-4f02-afe2-3927067cda87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T09:27:47.803109699Z\",\n      \"endTime\": \"2026-09-17T09:51:54.618583456Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sslv6ma6mn5k5mt36luuom09289pq58 still exists (deletion dispatched at 2026-09-17T09:27:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"724c2a9602b01afbc546852f539ff27b105da0871a0374f448eae350e85e32ce","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a9f77fe91a65ae001825b7e195168037cff1cc1fd2f1b2a967813a5f71db56d2","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f92288\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-djgr48qrs9vr\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\",\n      \"name\": \"a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:36:21.892022588Z\",\n      \"endTime\": \"2026-09-16T10:00:22.50023585Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss12t13cvqgvb69duuok3m0c1if4g29 still exists (deletion dispatched at 2026-09-16T09:36:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f92288\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-djgr48qrs9vr\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\",\n      \"name\": \"a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:36:21.892022588Z\",\n      \"endTime\": \"2026-09-16T10:00:22.50023585Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss12t13cvqgvb69duuok3m0c1if4g29 still exists (deletion dispatched at 2026-09-16T09:36:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"f25ee5738903cfa5e80de14eb219fe7a6033dded2da21704c137087674937bf8","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b2d9bdf10bd756a41af895df7f16c38d9b8ca7729eae2184e9c4375fba7db5fb","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012b4030\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-sk5lk2xqrw7v\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/234e26dc-5c76-4335-9749-4a1c49aae81e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/234e26dc-5c76-4335-9749-4a1c49aae81e\",\n      \"name\": \"234e26dc-5c76-4335-9749-4a1c49aae81e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:00:40.136241522Z\",\n      \"endTime\": \"2026-09-16T17:24:42.83613595Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7is141l1pe3gtmk90b2cej7ltp3gn still exists (deletion dispatched at 2026-09-16T17:00:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012b4030\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-sk5lk2xqrw7v\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/234e26dc-5c76-4335-9749-4a1c49aae81e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/234e26dc-5c76-4335-9749-4a1c49aae81e\",\n      \"name\": \"234e26dc-5c76-4335-9749-4a1c49aae81e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:00:40.136241522Z\",\n      \"endTime\": \"2026-09-16T17:24:42.83613595Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7is141l1pe3gtmk90b2cej7ltp3gn still exists (deletion dispatched at 2026-09-16T17:00:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"5cf9c85c408d35be4deef4e7c087c454f1070d1509c8652940e0ebeb5471037c","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f3bc88c6c8876f1c2ea5668dd924adc7c81d6e0349b9850342e67442ed3fe914","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00108c078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mzfrqt8tf9wq\" to finish deleting: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\",\n      \"name\": \"e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T11:18:42.744808437Z\",\n      \"endTime\": \"2026-09-16T11:42:49.617012035Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00108c078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mzfrqt8tf9wq\" to finish deleting: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\",\n      \"name\": \"e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T11:18:42.744808437Z\",\n      \"endTime\": \"2026-09-16T11:42:49.617012035Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"874e6ea76f2ba4acc89e73ac761bf0c81c038b75f11db73023de66c025986ab7","pr_number":6791,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0015da030\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2vshj8kd4ngg\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b25208f-0e8c-46c2-96d1-92647ed090d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b25208f-0e8c-46c2-96d1-92647ed090d2\",\n      \"name\": \"2b25208f-0e8c-46c2-96d1-92647ed090d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T09:42:31.310516975Z\",\n      \"endTime\": \"2026-09-17T10:06:34.577233142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0015da030\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2vshj8kd4ngg\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b25208f-0e8c-46c2-96d1-92647ed090d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b25208f-0e8c-46c2-96d1-92647ed090d2\",\n      \"name\": \"2b25208f-0e8c-46c2-96d1-92647ed090d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T09:42:31.310516975Z\",\n      \"endTime\": \"2026-09-17T10:06:34.577233142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"b8fb61f35748687f1492367050e150cb136907cb120fe4d9a22bd956f2ad9b80","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"82354ec7b175adc5a536e81d482a89743bf21cd916bcbb26d7a8a4f61c740937","failure_pattern":"ERROR CODE: InvalidTemplateDeployment; provider Microsoft.Authorization","search_query":"ERROR CODE: InvalidTemplateDeployment","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/regional/adminapi-alert-rule-role","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/regional/adminapi-alert-rule-role","occurrences":1}],"full_error_samples":["time=2026-09-19T00:59:12.578Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=regional step=adminapi-alert-rule-role description=\"Step adminapi-alert-rule-role\\n  Kind: ARM\\n  Template: templates/adminapi-alert-processing-rule-role.bicep\\n  Parameters: configurations/adminapi-alert-processing-rule-role.tmpl.bicepparam\"\ntime=2026-09-19T00:59:12.962Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=regional step=adminapi-alert-rule-role\ntime=2026-09-19T00:59:13.303Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=regional step=adminapi-alert-rule-role err=\"failed to run ARM step: failed to create deployment: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/73540b7e4361f329d8cc7347416ba379091383d8ff0e1f7fb10b7cc1abb08e96\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidTemplateDeployment\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidTemplateDeployment\\\",\\n    \\\"message\\\": \\\"The template deployment failed with error: 'Authorization failed for template resource '964da0aa-1cca-5874-beb3-3734babb6fde' of type 'Microsoft.Authorization/roleDefinitions'. The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id 'c209f8df-52ae-48fb-98ea-380f58b04652' does not have permission to perform action 'Microsoft.Authorization/roleDefinitions/write' at scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Authorization/roleDefinitions/964da0aa-1cca-5874-beb3-3734babb6fde'.'.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101109159930892288","occurred_at":"2026-09-19T00:40:35Z","signature_id":"db00f33af760674668e808df0b4cac804d0e009e1111194f83bb88e08459bd92","pr_number":7020,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"27391593c89e1b6a3153b058ef31be166dd830240fb93e3caa733a62c02edc05","failure_pattern":"ERROR CODE: ResourceCollectionRequestsThrottled; provider Microsoft.Network","search_query":"ERROR CODE: ResourceCollectionRequestsThrottled","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:141]: failed to find private KAS internal LB IP in managed resource group \"private-kas-pzzdfb6jcgvd--managed\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001100480\u003e: \n    failed to list load balancers in \"private-kas-pzzdfb6jcgvd--managed\": GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-pzzdfb6jcgvd--managed/providers/Microsoft.Network/loadBalancers\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: ResourceCollectionRequestsThrottled\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceCollectionRequestsThrottled\",\n        \"message\": \"Operation 'Microsoft.Network/loadBalancers/read' failed as server encountered too many requests. Please try after '1' seconds. Tracking Id is '0bca3dc6-65f8-40d2-be6e-d4168ed4b62f'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:141]: failed to find private KAS internal LB IP in managed resource group \"private-kas-pzzdfb6jcgvd--managed\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001100480\u003e: \n    failed to list load balancers in \"private-kas-pzzdfb6jcgvd--managed\": GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-pzzdfb6jcgvd--managed/providers/Microsoft.Network/loadBalancers\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: ResourceCollectionRequestsThrottled\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceCollectionRequestsThrottled\",\n        \"message\": \"Operation 'Microsoft.Network/loadBalancers/read' failed as server encountered too many requests. Please try after '1' seconds. Tracking Id is '0bca3dc6-65f8-40d2-be6e-d4168ed4b62f'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488796448493568","occurred_at":"2026-09-17T07:35:28Z","signature_id":"dcc1d91a022ed0888281ddf9849c28aaaf8b8a73e6c45ef734d0a5c17a44b021","pr_number":6856,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"750410b055b83de5666e72039e788f64cc42e64b3191a432c4cf3efab4dc693f","failure_pattern":"Operation cannot be fulfilled on nodes \"\u003cnode\u003e\": the object has been modified; please apply your changes to the latest version and try again","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node k6x3v2v7z6d4m6q-cordon-np-7ph5q-cjmjj\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000ac2640\u003e: \n    Operation cannot be fulfilled on nodes \"k6x3v2v7z6d4m6q-cordon-np-7ph5q-cjmjj\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node k6x3v2v7z6d4m6q-cordon-np-7ph5q-cjmjj\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000ac2640\u003e: \n    Operation cannot be fulfilled on nodes \"k6x3v2v7z6d4m6q-cordon-np-7ph5q-cjmjj\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","occurred_at":"2026-09-20T09:53:04Z","signature_id":"39cf09c4d4c3e066082fd6d0301325efef03f8489b16e7088e47c2ad9b88bbaa","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1","failure_pattern":"UpdateHCPCluster20260901 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w7nhqllrvzc4\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-w7nhqllrvzc4, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:184]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ca8a20\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w7nhqllrvzc4\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-w7nhqllrvzc4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:184]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ca8a20\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w7nhqllrvzc4\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-w7nhqllrvzc4, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101647291294158848","occurred_at":"2026-09-20T12:18:55Z","signature_id":"e0ec91f3cf6a62bae5b4c3d56509ed71730ec1128a9898d0313dd55a382d4fa9","pr_number":7025,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"5cfe4f61cfcfa6903321f1c971ea5ab31ebee2ed754015e96c99b68dfe286ebe","failure_pattern":"VM run command \"e2e-runcommand-1789704441845701237\" on VM \"\u003cvm\u003e\" failed and cleanup also failed: Get \"\u003curl\u003e\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"\u003curl\u003e\": stream error: stream ID 33; CANCEL; received from peer)","search_query":"VM run command \"e2e-runcommand-1789704441845701237\" on VM \"cidr-connectivity-test-test-vm\" failed and cleanup also failed: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/locations/CanadaCentral/operations/67651689-75ab-4f6c-aa6e-1746fbbdc2ad?p=c6bb2de4-260b-49a6-86a7-56c69bdc0403\u0026api-version=2024-03-01\u0026t=639253012427558463\u0026c=MIIHlDCCBnygAwIBAgIQTJW7e7XbMoQlWPa331WcnDANBgkqhkiG9w0BAQsFADA2MTQwMgYDVQQDEytDQ01FIEcxIFRMUyBSU0EgMjA0OCBTSEEyNTYgMjA0OSBXVVMyIENBIDAxMB4XDTI2MDgxMTE3MzMyMloXDTI3MDIwNjIzMzMyMlowQDE-MDwGA1UEAxM1YXN5bmNvcGVyYXRpb25zaWduaW5nY2VydGlmaWNhdGUubWFuYWdlbWVudC5henVyZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeQHh5tmZ6ZYRlf_Q_PwPJqUN9t1MEgnet-7jMlsIvkZBhjMDVPNEFS1UtIM-3StxxWCffcghG31P1UcexhHelJCujRWSZYWfEIC7z_S8BBwxDhcc2peEIdNKD1O6UKvglkD8KRsPxRT-ZwQYY3qFNngixnx3EDhtWr1a9Kg46bdJJAIIzu3flYJzQzP3yVrFuRQn4iH75UvhgEByfjJaD75Ugw9JYkgO52nE5dVc2ddxoRMWzJKxp_V2Adp7VqTbMx2T8hO9PWqkltIPwoAXIubXFj3Z5A6Wj7hE7XmY_2cFJKJfspjiqwLyfnLqlv9pCLhIVdG31g8DAGZM7fc2lAgMBAAGjggSSMIIEjjCBnQYDVR0gBIGVMIGSMAwGCisGAQQBgjd7AQEwZgYKKwYBBAGCN3sCAjBYMFYGCCsGAQUFBwICMEoeSAAzADMAZQAwADEAOQAyADEALQA0AGQANgA0AC0ANABmADgAYwAtAGEAMAA1ADUALQA1AGIAZABhAGYAZgBkADUAZQAzADMAZDAMBgorBgEEAYI3ewMCMAwGCisGAQQBgjd7BAIwDAYDVR0TAQH_BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDgYDVR0PAQH_BAQDAgWgMB0GA1UdDgQWBBQbwB3CifQh36XI9i1lBe8503G1RTAfBgNVHSMEGDAWgBSs43L6A7Jznj2VyO-HW67dG6HtaDCCAbIGA1UdHwSCAakwggGlMGmgZ6BlhmNodHRwOi8vcHJpbWFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwa6BpoGeGZWh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3R1czIvY3Jscy9jY21ld2VzdHVzMnBraS9jY21ld2VzdHVzMmljYTAxLzY3L2N1cnJlbnQuY3JsMFqgWKBWhlRodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwb6BtoGuGaWh0dHA6Ly9jY21ld2VzdHVzMnBraS53ZXN0dXMyLnBraS5jb3JlLndpbmRvd3MubmV0L2NlcnRpZmljYXRlQXV0aG9yaXRpZXMvY2NtZXdlc3R1czJpY2EwMS82Ny9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jYWNlcnRzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWV3ZXN0dXMycGtpLndlc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQACHqTwaTHdHWv9uHzVkq0ltJxzjLPIQHJnP6ndMrFnY-TIxQDrABl7eSPlINC2qk1CFExKLmjYM8aDst83BXP-yH0AHIvBrOGVdZvqFE-8KPpkqE97J5NNLNr1lLkz4vWpbjmJhLvYmureoMJjtcLpQyz-EfACVM90Lkmu8ueIkp5mjzxa_-cMa3KYwbFXLO8huXYcrhLhvs31Z9czJKjlPptL0ZE1-rBlMXGqTdZUE1CfPRoTjMMRZe_jEJYNKyxhL_5fT2mLxdlYJsMDeTU5qM-x3lA74zxVrUvC0ZvCQdK629aZQUbWV74Uzk1DmAtUiDhGvnsOW-rug_uLr621\u0026s=2ED070N0ZTuw-f1HVDJ70D4TL-E7a_pmemS26JAcsDWdtZNt5piI-6t7rp4DAeUvveysVwPzvAf5ULh-g26b96AAV9pXZ4I1vo74l2qjDBTA1ho2Ia11r_3U1XAtiPoLjGufya8CYXOek1iwZu7oaFEZ-4uayFIei-jUkQWxZ9Jz8_HKfyzJZZA1qJtJWKMIu_-fMQRELNXCWOkBgzgrUj1MP39GcY4oa5lZoVmPJGhHGseHZ--ETS4Vun62lwfxl7XrPb2PN0L9g7S07Nil12FIWSbh_qVJ2DC5m-7fPlkYze7LvocmWvHPwMOLiIbvvBS9Ui4BX7syQlC25zGKXw\u0026h=siGPFN1sMGFTEzhJalRHQiDtbRbqB8eZGXQRBJUWkyo\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qhh4b47m8ncv/providers/Microsoft.Compute/virtualMachines/cidr-connectivity-test-test-vm/runCommands/e2e-runcommand-1789704441845701237?api-version=2024-03-01\": stream error: stream ID 33; CANCEL; received from peer)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:293]: failed to create cluster role binding for external auth via VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d173e0\u003e: \n    VM run command \"e2e-runcommand-1789704441845701237\" on VM \"cidr-connectivity-test-test-vm\" failed and cleanup also failed: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/locations/CanadaCentral/operations/67651689-75ab-4f6c-aa6e-1746fbbdc2ad?p=c6bb2de4-260b-49a6-86a7-56c69bdc0403\u0026api-version=2024-03-01\u0026t=639253012427558463\u0026c=MIIHlDCCBnygAwIBAgIQTJW7e7XbMoQlWPa331WcnDANBgkqhkiG9w0BAQsFADA2MTQwMgYDVQQDEytDQ01FIEcxIFRMUyBSU0EgMjA0OCBTSEEyNTYgMjA0OSBXVVMyIENBIDAxMB4XDTI2MDgxMTE3MzMyMloXDTI3MDIwNjIzMzMyMlowQDE-MDwGA1UEAxM1YXN5bmNvcGVyYXRpb25zaWduaW5nY2VydGlmaWNhdGUubWFuYWdlbWVudC5henVyZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeQHh5tmZ6ZYRlf_Q_PwPJqUN9t1MEgnet-7jMlsIvkZBhjMDVPNEFS1UtIM-3StxxWCffcghG31P1UcexhHelJCujRWSZYWfEIC7z_S8BBwxDhcc2peEIdNKD1O6UKvglkD8KRsPxRT-ZwQYY3qFNngixnx3EDhtWr1a9Kg46bdJJAIIzu3flYJzQzP3yVrFuRQn4iH75UvhgEByfjJaD75Ugw9JYkgO52nE5dVc2ddxoRMWzJKxp_V2Adp7VqTbMx2T8hO9PWqkltIPwoAXIubXFj3Z5A6Wj7hE7XmY_2cFJKJfspjiqwLyfnLqlv9pCLhIVdG31g8DAGZM7fc2lAgMBAAGjggSSMIIEjjCBnQYDVR0gBIGVMIGSMAwGCisGAQQBgjd7AQEwZgYKKwYBBAGCN3sCAjBYMFYGCCsGAQUFBwICMEoeSAAzADMAZQAwADEAOQAyADEALQA0AGQANgA0AC0ANABmADgAYwAtAGEAMAA1ADUALQA1AGIAZABhAGYAZgBkADUAZQAzADMAZDAMBgorBgEEAYI3ewMCMAwGCisGAQQBgjd7BAIwDAYDVR0TAQH_BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDgYDVR0PAQH_BAQDAgWgMB0GA1UdDgQWBBQbwB3CifQh36XI9i1lBe8503G1RTAfBgNVHSMEGDAWgBSs43L6A7Jznj2VyO-HW67dG6HtaDCCAbIGA1UdHwSCAakwggGlMGmgZ6BlhmNodHRwOi8vcHJpbWFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwa6BpoGeGZWh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3R1czIvY3Jscy9jY21ld2VzdHVzMnBraS9jY21ld2VzdHVzMmljYTAxLzY3L2N1cnJlbnQuY3JsMFqgWKBWhlRodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwb6BtoGuGaWh0dHA6Ly9jY21ld2VzdHVzMnBraS53ZXN0dXMyLnBraS5jb3JlLndpbmRvd3MubmV0L2NlcnRpZmljYXRlQXV0aG9yaXRpZXMvY2NtZXdlc3R1czJpY2EwMS82Ny9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jYWNlcnRzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWV3ZXN0dXMycGtpLndlc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQACHqTwaTHdHWv9uHzVkq0ltJxzjLPIQHJnP6ndMrFnY-TIxQDrABl7eSPlINC2qk1CFExKLmjYM8aDst83BXP-yH0AHIvBrOGVdZvqFE-8KPpkqE97J5NNLNr1lLkz4vWpbjmJhLvYmureoMJjtcLpQyz-EfACVM90Lkmu8ueIkp5mjzxa_-cMa3KYwbFXLO8huXYcrhLhvs31Z9czJKjlPptL0ZE1-rBlMXGqTdZUE1CfPRoTjMMRZe_jEJYNKyxhL_5fT2mLxdlYJsMDeTU5qM-x3lA74zxVrUvC0ZvCQdK629aZQUbWV74Uzk1DmAtUiDhGvnsOW-rug_uLr621\u0026s=2ED070N0ZTuw-f1HVDJ70D4TL-E7a_pmemS26JAcsDWdtZNt5piI-6t7rp4DAeUvveysVwPzvAf5ULh-g26b96AAV9pXZ4I1vo74l2qjDBTA1ho2Ia11r_3U1XAtiPoLjGufya8CYXOek1iwZu7oaFEZ-4uayFIei-jUkQWxZ9Jz8_HKfyzJZZA1qJtJWKMIu_-fMQRELNXCWOkBgzgrUj1MP39GcY4oa5lZoVmPJGhHGseHZ--ETS4Vun62lwfxl7XrPb2PN0L9g7S07Nil12FIWSbh_qVJ2DC5m-7fPlkYze7LvocmWvHPwMOLiIbvvBS9Ui4BX7syQlC25zGKXw\u0026h=siGPFN1sMGFTEzhJalRHQiDtbRbqB8eZGXQRBJUWkyo\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qhh4b47m8ncv/providers/Microsoft.Compute/virtualMachines/cidr-connectivity-test-test-vm/runCommands/e2e-runcommand-1789704441845701237?api-version=2024-03-01\": stream error: stream ID 33; CANCEL; received from peer)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:293]: failed to create cluster role binding for external auth via VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d173e0\u003e: \n    VM run command \"e2e-runcommand-1789704441845701237\" on VM \"cidr-connectivity-test-test-vm\" failed and cleanup also failed: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/locations/CanadaCentral/operations/67651689-75ab-4f6c-aa6e-1746fbbdc2ad?p=c6bb2de4-260b-49a6-86a7-56c69bdc0403\u0026api-version=2024-03-01\u0026t=639253012427558463\u0026c=MIIHlDCCBnygAwIBAgIQTJW7e7XbMoQlWPa331WcnDANBgkqhkiG9w0BAQsFADA2MTQwMgYDVQQDEytDQ01FIEcxIFRMUyBSU0EgMjA0OCBTSEEyNTYgMjA0OSBXVVMyIENBIDAxMB4XDTI2MDgxMTE3MzMyMloXDTI3MDIwNjIzMzMyMlowQDE-MDwGA1UEAxM1YXN5bmNvcGVyYXRpb25zaWduaW5nY2VydGlmaWNhdGUubWFuYWdlbWVudC5henVyZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeQHh5tmZ6ZYRlf_Q_PwPJqUN9t1MEgnet-7jMlsIvkZBhjMDVPNEFS1UtIM-3StxxWCffcghG31P1UcexhHelJCujRWSZYWfEIC7z_S8BBwxDhcc2peEIdNKD1O6UKvglkD8KRsPxRT-ZwQYY3qFNngixnx3EDhtWr1a9Kg46bdJJAIIzu3flYJzQzP3yVrFuRQn4iH75UvhgEByfjJaD75Ugw9JYkgO52nE5dVc2ddxoRMWzJKxp_V2Adp7VqTbMx2T8hO9PWqkltIPwoAXIubXFj3Z5A6Wj7hE7XmY_2cFJKJfspjiqwLyfnLqlv9pCLhIVdG31g8DAGZM7fc2lAgMBAAGjggSSMIIEjjCBnQYDVR0gBIGVMIGSMAwGCisGAQQBgjd7AQEwZgYKKwYBBAGCN3sCAjBYMFYGCCsGAQUFBwICMEoeSAAzADMAZQAwADEAOQAyADEALQA0AGQANgA0AC0ANABmADgAYwAtAGEAMAA1ADUALQA1AGIAZABhAGYAZgBkADUAZQAzADMAZDAMBgorBgEEAYI3ewMCMAwGCisGAQQBgjd7BAIwDAYDVR0TAQH_BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDgYDVR0PAQH_BAQDAgWgMB0GA1UdDgQWBBQbwB3CifQh36XI9i1lBe8503G1RTAfBgNVHSMEGDAWgBSs43L6A7Jznj2VyO-HW67dG6HtaDCCAbIGA1UdHwSCAakwggGlMGmgZ6BlhmNodHRwOi8vcHJpbWFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwa6BpoGeGZWh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3R1czIvY3Jscy9jY21ld2VzdHVzMnBraS9jY21ld2VzdHVzMmljYTAxLzY3L2N1cnJlbnQuY3JsMFqgWKBWhlRodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwb6BtoGuGaWh0dHA6Ly9jY21ld2VzdHVzMnBraS53ZXN0dXMyLnBraS5jb3JlLndpbmRvd3MubmV0L2NlcnRpZmljYXRlQXV0aG9yaXRpZXMvY2NtZXdlc3R1czJpY2EwMS82Ny9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jYWNlcnRzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWV3ZXN0dXMycGtpLndlc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQACHqTwaTHdHWv9uHzVkq0ltJxzjLPIQHJnP6ndMrFnY-TIxQDrABl7eSPlINC2qk1CFExKLmjYM8aDst83BXP-yH0AHIvBrOGVdZvqFE-8KPpkqE97J5NNLNr1lLkz4vWpbjmJhLvYmureoMJjtcLpQyz-EfACVM90Lkmu8ueIkp5mjzxa_-cMa3KYwbFXLO8huXYcrhLhvs31Z9czJKjlPptL0ZE1-rBlMXGqTdZUE1CfPRoTjMMRZe_jEJYNKyxhL_5fT2mLxdlYJsMDeTU5qM-x3lA74zxVrUvC0ZvCQdK629aZQUbWV74Uzk1DmAtUiDhGvnsOW-rug_uLr621\u0026s=2ED070N0ZTuw-f1HVDJ70D4TL-E7a_pmemS26JAcsDWdtZNt5piI-6t7rp4DAeUvveysVwPzvAf5ULh-g26b96AAV9pXZ4I1vo74l2qjDBTA1ho2Ia11r_3U1XAtiPoLjGufya8CYXOek1iwZu7oaFEZ-4uayFIei-jUkQWxZ9Jz8_HKfyzJZZA1qJtJWKMIu_-fMQRELNXCWOkBgzgrUj1MP39GcY4oa5lZoVmPJGhHGseHZ--ETS4Vun62lwfxl7XrPb2PN0L9g7S07Nil12FIWSbh_qVJ2DC5m-7fPlkYze7LvocmWvHPwMOLiIbvvBS9Ui4BX7syQlC25zGKXw\u0026h=siGPFN1sMGFTEzhJalRHQiDtbRbqB8eZGXQRBJUWkyo\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qhh4b47m8ncv/providers/Microsoft.Compute/virtualMachines/cidr-connectivity-test-test-vm/runCommands/e2e-runcommand-1789704441845701237?api-version=2024-03-01\": stream error: stream ID 33; CANCEL; received from peer)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780897212043264","occurred_at":"2026-09-18T02:56:11Z","signature_id":"66f62a5d5d02e16dfffeec66474c962d8c14552e10473a54125784861158f0d5","pr_number":6898,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"55fc315450b35ec48de3526d899619e34f6aae1d138260064fe2d53d5403f86e","failure_pattern":"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.006s: DaemonSet \u003cnamespace\u003e/global-pull-secret-syncer has no desired pods scheduled","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:183]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010007a0\u003e: \n    VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.006s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:183]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010007a0\u003e: \n    VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.006s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"f61d48f7e4dfc0d5ed670d4815de996c6e11f3ee6013821737755d17d338fd53","pr_number":7032,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7855bddb074e5ee4da59762c697f802a687000bfd9477e9f92fc724e190b9330","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to get pod logs for pod fips-check-\u003cid\u003e in namespace e2e-fips-test-\u003cid\u003e on node \u003cnode\u003e after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv: failed to get pod logs for pod fips-check-rxfv5 in namespace e2e-fips-test-k5rlj on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-rxfv5)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc0014ee660\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv: failed to get pod logs for pod fips-check-rxfv5 in namespace e2e-fips-test-k5rlj on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-rxfv5)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc0014ee660\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv: failed to get pod logs for pod fips-check-rxfv5 in namespace e2e-fips-test-k5rlj on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-rxfv5)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569759195074560","occurred_at":"2026-09-17T12:57:11Z","signature_id":"b4cc553a2220034e450c979876c39e29fdc1278faa5ca56fae473d7a07ad2b7c","pr_number":6989,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f98a5615cab85c5989e6a0ac742f663ddff170e09cb4f716bf9a5978a279cf45","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to get pod logs for pod fips-check-psxxs in namespace e2e-fips-test-\u003cid\u003e on node \u003cnode\u003e after retries: an error on the server (\"unknown\") has prevented the request from succeeding","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5: failed to get pod logs for pod fips-check-psxxs in namespace e2e-fips-test-rx7hl on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-psxxs)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc002923158\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5: failed to get pod logs for pod fips-check-psxxs in namespace e2e-fips-test-rx7hl on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-psxxs)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc002923158\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5: failed to get pod logs for pod fips-check-psxxs in namespace e2e-fips-test-rx7hl on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-psxxs)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"64d1fa05ea54883aa2363d63f6b4eef64e006023913f4dea31d5e86c98373938","pr_number":6805,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc002e66048\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc002e66048\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","signature_id":"dcd0bfb37a59db9bd0b774c26b5a47b2e4002b0f9d6d7d9a6292d0c9dca75bea","pr_number":6966,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"27efdba0688c010a9407237cceb9d29c447c7f9bd7965cf5f46a9ce24313bf72","failure_pattern":"alert [hcp] userJourneySwiftRouterAvailability1h5m fired","search_query":"alert [hcp] userJourneySwiftRouterAvailability1h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneySwiftRouterAvailability1h5m does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneySwiftRouterAvailability1h5m does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-19T01:16:42Z\n  Ended:       2026-09-19T01:52:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneySwiftRouterAvailability1h5m\", burn_rate_tier=\"fast\", cluster=\"ci01-j8883328-mgmt-1\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8883328/providers/microsoft.monitor/accounts/hcps-j8883328\", region=\"westus3\", severity=\"3\", short_window=\"5m\"\n  Description: SWIFT router-capacity availability is consuming its error budget at the fast burn rate.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-19T01:38:43Z\n  Ended:       2026-09-19T01:47:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneySwiftRouterAvailability1h5m\", burn_rate_tier=\"fast\", cluster=\"ci01-j8883328-mgmt-2\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8883328/providers/microsoft.monitor/accounts/hcps-j8883328\", region=\"westus3\", severity=\"3\", short_window=\"5m\"\n  Description: SWIFT router-capacity availability is consuming its error budget at the fast burn rate."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101105304358883328","occurred_at":"2026-09-19T00:25:15Z","signature_id":"7f5ecf7c1071938ce4c9e9eac6684e37aa439774f897cb1f14f73cd8fcb2bfae","pr_number":6359,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7fa847820ad43185d68e54ccb17caf15404b33f7f9865c668f2186c67969315a","failure_pattern":"alert [hcp] userJourneySwiftRouterAvailability6h30m fired","search_query":"alert [hcp] userJourneySwiftRouterAvailability6h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneySwiftRouterAvailability6h30m does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneySwiftRouterAvailability6h30m does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-19T01:39:45Z\n  Ended:       2026-09-19T01:51:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneySwiftRouterAvailability6h30m\", burn_rate_tier=\"medium\", cluster=\"ci01-j8883328-mgmt-2\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8883328/providers/microsoft.monitor/accounts/hcps-j8883328\", region=\"westus3\", severity=\"3\", short_window=\"30m\"\n  Description: SWIFT router-capacity availability is consuming its error budget at the medium burn rate.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-19T01:41:44Z\n  Ended:       2026-09-19T01:56:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneySwiftRouterAvailability6h30m\", burn_rate_tier=\"medium\", cluster=\"ci01-j8883328-mgmt-1\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8883328/providers/microsoft.monitor/accounts/hcps-j8883328\", region=\"westus3\", severity=\"3\", short_window=\"30m\"\n  Description: SWIFT router-capacity availability is consuming its error budget at the medium burn rate."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101105304358883328","occurred_at":"2026-09-19T00:25:15Z","signature_id":"b184a85bf7c0152d493e5f2ffc42f651c7eaa05c819cf7f3318de6734bf20ad7","pr_number":6359,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c4e2bd93d0410f48ddfa390cc77604492607a58b2a4eaff204baa6c7bed3af07","failure_pattern":"alert [svc] KubeContainerOOMKilled fired","search_query":"alert [svc] KubeContainerOOMKilled fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T13:42:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j4979456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.70:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4979456/providers/microsoft.monitor/accounts/services-j4979456\", namespace=\"arobit\", pod=\"arobit-forwarder-k26lv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9061149b-822a-4131-9e5f-b554ee1870de\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-k26lv on cluster ci01-j4979456-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T13:42:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j4979456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.70:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4979456/providers/microsoft.monitor/accounts/services-j4979456\", namespace=\"arobit\", pod=\"arobit-forwarder-k26lv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9061149b-822a-4131-9e5f-b554ee1870de\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-k26lv on cluster ci01-j4979456-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554708664979456","occurred_at":"2026-09-17T11:57:12Z","signature_id":"6db51492a3864e98312b98dbacda80d9a85398c3e3f34314664719d56a168f65","pr_number":6756,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a89f0d2edcd262580a6fd914503ced16ad52ae763f2c2255b4f5af55d375ecd3","failure_pattern":"alert [svc] KubeContainerWaiting fired","search_query":"alert [svc] KubeContainerWaiting fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerWaiting does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerWaiting does not fire","occurrences":1}],"full_error_samples":["alert fired 5 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:30:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"secrets-store\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-cwxxr\", region=\"canadacentral\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-cwxxr in namespace kube-system on container secrets-store has been in waiting state for longer than 1 hour.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:30:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"liveness-probe\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-cwxxr\", region=\"canadacentral\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-cwxxr in namespace kube-system on container liveness-probe has been in waiting state for longer than 1 hour.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:30:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"node-driver-registrar\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-cwxxr\", region=\"canadacentral\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-cwxxr in namespace kube-system on container node-driver-registrar has been in waiting state for longer than 1 hour.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:30:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", namespace=\"kube-system\", pod=\"ama-metrics-node-xfmjp\", region=\"canadacentral\", severity=\"warning\"\n  Description: pod/ama-metrics-node-xfmjp in namespace kube-system on container prometheus-collector has been in waiting state for longer than 1 hour.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:30:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"addon-token-adapter\", namespace=\"kube-system\", pod=\"ama-metrics-node-xfmjp\", region=\"canadacentral\", severity=\"warning\"\n  Description: pod/ama-metrics-node-xfmjp in namespace kube-system on container addon-token-adapter has been in waiting state for longer than 1 hour."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"97461ac292fa9fba61fa81d505ebf8a88d3f96ffd9a07f7d644245f3dd5aa26b","pr_number":6982,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T02:39:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T02:40:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T02:46:35Z\n  Ended:       2026-09-17T02:54:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-17T02:46:35Z\n  Ended:       2026-09-17T02:59:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7/klusterlet-addon-workmgr is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"4d675a2aa4e866465ba35a460544e188e20719f62123eb05ebda79f62db3498b","pr_number":6940,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d6c6b690ca60846dd98e464313d6a9672b9abd8744604edd915627eb68203f9d","failure_pattern":"alert [svc] KubeNodeNotReady fired","search_query":"alert [svc] KubeNodeNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeNotReady does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeNotReady does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:00:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"prometheus\", node=\"aks-system-28181016-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-system-28181016-vmss000000 has been unready for more than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:00:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"prometheus\", node=\"aks-system-28181016-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-system-28181016-vmss000000 has been unready for more than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"e58f291d243075248b87935d2709ed85f1b36fdb31f78f88aaae185cc551feae","pr_number":6982,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T21:40:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j0025600-mgmt-2\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100689397040025600","occurred_at":"2026-09-17T20:52:35Z","signature_id":"c8d2c94c46ea41bb6616321fc23a840d9b2ec094985319abefcb6af50ae87597","pr_number":6945,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d89bec3ea75e80fed983a698bbef1fbb78c33e77e634e6400cab6b214e710fe5","failure_pattern":"alert [svc] userJourneyFrontendLatencyP991h5m fired","search_query":"alert [svc] userJourneyFrontendLatencyP991h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP991h5m does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP991h5m does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T09:39:29Z\n  Ended:       2026-09-16T09:50:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatencyP991h5m\", cluster=\"ci01-j3290112-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3290112/providers/microsoft.monitor/accounts/services-j3290112\", severity=\"3\", short_window=\"5m\", slo=\"frontend-latency\"\n  Description: Frontend p99 latency on cluster ci01-j3290112-svc is above 5s on both the 1h and 5m windows (severe miss of the p99 \u003c 1s ARM sync SLO).\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-16T10:10:33Z\n  Ended:       2026-09-16T10:17:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatencyP991h5m\", cluster=\"ci01-j3290112-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3290112/providers/microsoft.monitor/accounts/services-j3290112\", severity=\"3\", short_window=\"5m\", slo=\"frontend-latency\"\n  Description: Frontend p99 latency on cluster ci01-j3290112-svc is above 5s on both the 1h and 5m windows (severe miss of the p99 \u003c 1s ARM sync SLO)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132038643290112","occurred_at":"2026-09-16T07:57:51Z","signature_id":"13f9b4b06b1b9850ff1976e03ab24843f0b045626fd218be7150c7dace6ffbef","pr_number":6503,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e9ea5f54afe3e98f1a64398661238c6fc0e342dcd1119b0d656cfccf60db80ec","failure_pattern":"breakglass session kubeconfig not ready: timeout waiting for session to become ready; HostedControlPlane exists but is not ready","search_query":"failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-wlfqkcdjh9rm/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-lbdxk/kubeconfig: timeout waiting for session to become ready (last status: {\"HostedControlPlaneAvailable\":{\"message\":\"HostedControlPlane exists but is not ready\",\"reason\":\"HostedControlPlaneNotReady\",\"status\":\"False\"},\"status\":\"Session is not ready\"})","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:185]: failed to create SRE breakglass credentials with aro-sre-pso permissions\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a7f820\u003e: \n    failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-wlfqkcdjh9rm/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-lbdxk/kubeconfig: timeout waiting for session to become ready (last status: {\"HostedControlPlaneAvailable\":{\"message\":\"HostedControlPlane exists but is not ready\",\"reason\":\"HostedControlPlaneNotReady\",\"status\":\"False\"},\"status\":\"Session is not ready\"})\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:185]: failed to create SRE breakglass credentials with aro-sre-pso permissions\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a7f820\u003e: \n    failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-wlfqkcdjh9rm/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-lbdxk/kubeconfig: timeout waiting for session to become ready (last status: {\"HostedControlPlaneAvailable\":{\"message\":\"HostedControlPlane exists but is not ready\",\"reason\":\"HostedControlPlaneNotReady\",\"status\":\"False\"},\"status\":\"Session is not ready\"})\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100153419804184576","occurred_at":"2026-09-16T09:22:48Z","signature_id":"6c4e73f54e8d63150bdff29f74282ba9a9e2ff1e60f7bd58d45e6cacc3dd7869","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fac5ab822da5a4db89037217494583b4069bba01be61d8f3a5122bc721b483e9","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"0IciLMVmLbqimBoU8f5q6QBEpmWjuJNgDzWaCM2v9MM=\"","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"0IciLMVmLbqimBoU8f5q6QBEpmWjuJNgDzWaCM2v9MM=\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001ffe8a0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"0IciLMVmLbqimBoU8f5q6QBEpmWjuJNgDzWaCM2v9MM=\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001ffe8a0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"0IciLMVmLbqimBoU8f5q6QBEpmWjuJNgDzWaCM2v9MM=\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252538870697984","occurred_at":"2026-09-16T15:56:40Z","signature_id":"b7562eb1317f094fe7e5e1b3fd0e3859a05304435b1ff984fb6d716f402b4c7c","pr_number":6791,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1ffc144947159d20ed7dd5010cb0a93b810641281a5d037d76b1920de5d6d537","failure_pattern":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later","search_query":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000b28b70\u003e: \n    clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000b28b70\u003e: \n    clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100747284332417024","occurred_at":"2026-09-18T00:42:37Z","signature_id":"82fd789c0354677fc94a578c2ebbcf48d32a60e2464d95dc9709afd5a7216752","pr_number":6972,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"422ddc32f327db7a61c881b4411ef7767ec79e98b46476bd6a0cffdccf708740","failure_pattern":"expected all nodes to have label 'key1=value1'","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"48929c5f392dda19c39d764a9d4117d26ea3b70f835a97462a0249c92cc9fdf4","pr_number":7032,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","failure_pattern":"expected node pool nodes to report at least one release image ref before downgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:753]: expected node pool nodes to report at least one release image ref before downgrade","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:753]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:753]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"31f7686d857db3ec2fac93987fae6b6a3245782f4b94f1b84b4b75f60d2d2c13","pr_number":7032,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d33373d20e5996675bec12509adcc946f8d57b13e0f18cc57679d39e08115eb4","failure_pattern":"failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"\u003cresource-group\u003e\": Get \"\u003curl\u003e\": stream error: stream ID 13; CANCEL; received from peer","search_query":"failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8?api-version=2021-04-01\": stream error: stream ID 13; CANCEL; received from peer","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d270f8\u003e: \n    failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8?api-version=2021-04-01\": stream error: stream ID 13; CANCEL; received from peer\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d270f8\u003e: \n    failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8?api-version=2021-04-01\": stream error: stream ID 13; CANCEL; received from peer\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569681449455616","occurred_at":"2026-09-17T12:56:50Z","signature_id":"2126196a18f3d500d8a52360e541d2fceb67e8a67c46b2636cf54189a72bfe35","pr_number":6157,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e86119606257db722a985501dec6db3353e9294ec584e08a2cf350342957ce78","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context canceled]","search_query":"failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: context canceled]","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n  \"name\": \"c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-18T03:56:03.231032121Z\",\n  \"endTime\": \"2026-09-18T03:56:41.678977419Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: context canceled]\nExpected\n    \u003c[]error | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n  \"name\": \"c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-18T03:56:03.231032121Z\",\n  \"endTime\": \"2026-09-18T03:56:41.678977419Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: context canceled]\nExpected\n    \u003c[]error | len:2, cap:2\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","signature_id":"de20ef80e43125ed23aee61f6adcfb8942fa81561feaefb4e80cef19ed4db1c7","pr_number":6972,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"eaacc54ebd46cefe15abd433d5c3a394c2c9eab33016bf077e0076e05e8a3f60","failure_pattern":"failed to create route: the server is currently unable to handle the request","search_query":"failed to create route: the server is currently unable to handle the request","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-v458r6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001029700\u003e: \n    failed to create route: the server is currently unable to handle the request\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-v458r6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001029700\u003e: \n    failed to create route: the server is currently unable to handle the request\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023746776109056","occurred_at":"2026-09-18T19:01:10Z","signature_id":"c0b449ac08413f5f7083190089ef0d839099fa2699d64d1c9f2f8d26abbef529","pr_number":7010,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"40d94d1862472748004cc33c4c935e1c5142aa767d3a1a077dd2606d1dc7c7d3","failure_pattern":"failed to roll out Helm release: Internal error occurred: failed calling webhook \"validation.istio.io\": failed to call webhook: Post \"\u003curl\u003e\": EOF","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","occurrences":1}],"full_error_samples":["time=2026-09-17T09:16:31.171Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-17T09:16:31.636Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:53204456-a0a3-4fd2-89f7-36eb569bbb5d cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-ci01-j2597888] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j2597888.hcpsvc.osadev.cloud] image:map[digest:sha256:dfb4a580c4ae075df76e82458246c28e24407e254922d6c888d0d2793d3c1737 registry:arohcpsvcdev.azurecr.io repository:ci-op-6i9x21qi/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-6i9x21qi/pipeline:pull workloadIdentityClientId:0ee3f5a0-d80b-4385-abcc-91385fcf6f70 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:15494d5f-cf01-4a47-ae22-344126602800 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-09-17T09:16:31.636Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-17T09:16:31.786Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-09-17T09:16:31.977Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-09-17T09:16:31.977Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-17T09:16:32.031Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-17T09:16:37.667Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\"\ntime=2026-09-17T09:16:37.667Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100504522802597888","occurred_at":"2026-09-17T08:37:58Z","signature_id":"bbb80a3447e86b156cad4ed50a4810d3fa341869a205ecefaab92d1964bbcc6f","pr_number":6990,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d31684d18713ffa0f31a08468a5622ddeecda759a6bb53e1d78d51a31e6f875b","failure_pattern":"failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/15 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/15\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-09-19T13:26:44.501Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 description=\"Step kubelet-ds\\n Kind: Helm\\n\"\ntime=2026-09-19T13:26:44.949Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 values=map[enabled:true]\ntime=2026-09-19T13:26:44.949Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1\ntime=2026-09-19T13:26:45.277Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 namespace=kubelet-ds\ntime=2026-09-19T13:26:45.468Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 dryRun=false\ntime=2026-09-19T13:26:45.468Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1\ntime=2026-09-19T13:26:45.524Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1\ntime=2026-09-19T13:31:46.698Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 err=\"resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/15\\ncontext deadline exceeded\"\ntime=2026-09-19T13:31:46.698Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/15\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7023/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101292731853705216","occurred_at":"2026-09-19T12:50:02Z","signature_id":"0842be46fe2b3ebe671974ce35c826abb71f7583993632c9a306d69aa8ced78e","pr_number":7023,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e45f8e323244cd43106961bb8120aab24b3a867987bf8529ce7b7b4509a092cc","failure_pattern":"failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/18 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/18\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-16T09:03:26.822Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 description=\"Step kubelet-ds\\n Kind: Helm\\n\"\ntime=2026-09-16T09:03:27.157Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 values=\"map[enabled:true kubeletParametersImage:mcr.microsoft.com/aks/command/runtime@sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23]\"\ntime=2026-09-16T09:03:27.157Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-09-16T09:03:27.284Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 namespace=kubelet-ds\ntime=2026-09-16T09:03:27.475Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 dryRun=false\ntime=2026-09-16T09:03:27.475Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-09-16T09:03:27.530Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-09-16T09:08:28.583Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/18\\ncontext deadline exceeded\"\ntime=2026-09-16T09:08:28.583Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/18\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132003708932096","occurred_at":"2026-09-16T07:57:42Z","signature_id":"4592cebb946834bf2f5597cc8f68b2f39cdbe688bce563e773bde0e8a997725a","pr_number":6009,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1a0fe5d7569a83b55b15427e5bd80f1ae31a9f5785e0f4d3715f223065550ec9","failure_pattern":"failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","occurrences":1}],"full_error_samples":["time=2026-09-18T21:55:47.666Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-18T21:55:48.312Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[armPermissionsManagerIdentity:map[certName:armHelperPoolCert-6 clientId:9c78a811-9c85-4eb0-9c84-d0843866c888 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpci01oidcj1086464.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpci01oidcj1086464.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] backupCadenceProfile:testing backupScheduleState:Enabled clusterScopedIdentitiesConfig:map[roleSetName:dev] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:780fc8ab-b71d-4dc8-a753-5e608208a59f databaseName:arohcpci01-rp-j1086464 databaseUrl:https://arohcpci01-rp-j1086464.documents.azure.com:443/ kubeApplierCosmosContainerName:Manifests-MC-1 location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-vh2hkbkg/pipeline@sha256:ec4f5311c601606f0bdcb4316ea9dff923b912c8e13a93addd5ac2c23dd9e86c replicas:2 resources:map[limits:map[memory:unlimited] requests:map[cpu:475m memory:1Gi]] zoneCount:3] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] logVerbosity:4 maestro:map[sourceEnvironmentIdentifier:arohcpci01] miMock:map[certName:msiMockPoolCert-12 clientId:50d052a6-7aec-48b9-b43e-b8692a8f0ead clientIdSecretName: principalId:a76148d4-cb11-4b93-9363-54ba8239b0b1 principalIdSecretName: tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-vh2hkbkg/pipeline:pull workloadIdentityClientId:09708096-ca85-4992-8eb5-667a9b9d3fce workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:780fc8ab-b71d-4dc8-a753-5e608208a59f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-09-18T21:55:48.312Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-18T21:55:48.698Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-09-18T21:55:48.964Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-09-18T21:55:48.964Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-18T21:55:49.102Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-18T22:00:55.361Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"\ntime=2026-09-18T22:00:55.361Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101059271281086464","occurred_at":"2026-09-18T21:22:20Z","signature_id":"b8972dd0ff9a3ca0c20207306d152fa4afd8e4eb6a37b8d1c512cde87ef5ae3e","pr_number":6577,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"dac8a1b25a3188542fe843437616ad526549332051ce7885378b1b60aa94182a","failure_pattern":"failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kube-events (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kube-events (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-09-18T20:01:01.932Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 description=\"Step kube-events\\n Kind: Helm\\n\"\ntime=2026-09-18T20:01:02.379Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 values=\"map[enabled:true image:map[digest:sha256:5177425b194b5d289664c62bedbdc95a42605bbfcde48fc7ddbb31bee0194b90 registry:kubernetesshared.azurecr.io repository:shared/kube-events] resources:map[limits:map[memory:unlimited] requests:map[cpu:200m memory:512Mi]]]\"\ntime=2026-09-18T20:01:02.379Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1\ntime=2026-09-18T20:01:02.574Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 namespace=monitoring\ntime=2026-09-18T20:01:02.761Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 dryRun=false\ntime=2026-09-18T20:01:02.761Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1\ntime=2026-09-18T20:01:02.827Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1\ntime=2026-09-18T20:06:05.062Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 err=\"resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-09-18T20:06:05.062Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023754950807552","occurred_at":"2026-09-18T19:01:12Z","signature_id":"30e287a059c7694cef227767592b23353845f4281b98b8c7ff98febc06270f17","pr_number":7002,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"35fcc68c27835aae14daafdef12fc813fb2e38cc3e35d2f974eb6cc333b3ce7e","failure_pattern":"kube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane","search_query":"kube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:85]: Timed out after 900.000s.\nkube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:83 with:\nexpected kube_node_info metrics for at least one hostedcontrolplane but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:85]: Timed out after 900.000s.\nkube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:83 with:\nexpected kube_node_info metrics for at least one hostedcontrolplane but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100556682735128576","occurred_at":"2026-09-17T12:05:14Z","signature_id":"7be65470254fc4c585209a09735c416c75334c6ead21b5d5328120538ae66b13","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cea54d0df4cc4df84a4bbaf41795235750a198ce743843fbdff1373606efa1aa","failure_pattern":"kube_node_info metrics never appeared in Azure Monitor for cluster \"\"","search_query":"kube_node_info metrics never appeared in Azure Monitor for cluster \"\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:97]: Timed out after 900.001s.\nkube_node_info metrics never appeared in Azure Monitor for cluster \"\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:95 with:\nexpected kube_node_info metrics for cluster \"\" but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:97]: Timed out after 900.001s.\nkube_node_info metrics never appeared in Azure Monitor for cluster \"\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:95 with:\nexpected kube_node_info metrics for cluster \"\" but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"4645b1f607eafec38a4f58b597cf9ee778993ba4ba3bc944eda97fc9f0f70da7","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"990ae447fc69c4a73fb4f72088b1a7f653aec54482f588061e215576813bf077","failure_pattern":"node pool \"\u003cnodepool\u003e\" should have at least 2 nodes (autoscaling min), found 0","search_query":"node pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:518]: Timed out after 900.001s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:505 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n    \u003cint\u003e: ...\nto be \u003e=\n    \u003cint\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:518]: Timed out after 900.001s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:505 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n    \u003cint\u003e: ...\nto be \u003e=\n    \u003cint\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"204e1750cfbcdccc7a1aa87cf8b29089bd7645257cbc1509ff43f3d1c64bf5f6","pr_number":7032,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"662e49bfb814d1960ee5fb9e525e6ffab8495cd359fc51176893943560eee29a","failure_pattern":"node pool \"\u003cnodepool\u003e\" should have exactly one active version after initial creation","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:144]: node pool \"np-1\" should have exactly one active version after initial creation","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:144]: node pool \"np-1\" should have exactly one active version after initial creation\nExpected\n    \u003c[]*armredhatopenshifthcp.NodePoolActiveVersion | len:0, cap:0\u003e: ...\nto have length 1\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:144]: node pool \"np-1\" should have exactly one active version after initial creation\nExpected\n    \u003c[]*armredhatopenshifthcp.NodePoolActiveVersion | len:0, cap:0\u003e: ...\nto have length 1"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222891458039808","occurred_at":"2026-09-16T13:58:52Z","signature_id":"4c1a786ccdf5f4a6714330ab65d6eaf1faaac9d21fed54613d33cde0056d6e52","pr_number":6952,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"62df4554fa8988de83bcea5588b7f5f69715e1db3483f50b3137f97138b59e77","failure_pattern":"run command \"e2e-runcommand-1789636815233707722\" failed with exit code 28 on VM \"\u003cvm\u003e\" (output: \"000\")","search_query":"run command \"e2e-runcommand-1789636815233707722\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13736263736263737,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 654.956s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc00277e080\u003e: \n    run command \"e2e-runcommand-1789636815233707722\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 654.956s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc00277e080\u003e: \n    run command \"e2e-runcommand-1789636815233707722\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494074514509824","occurred_at":"2026-09-17T07:56:27Z","signature_id":"1395a5c67402b5879cd22d694e49480a23894641a58ba386a295937a83eff76f","pr_number":6752,"after_last_push_of_merged_pr":true}]}]},{"environment":"int","summary":{"total_runs":24,"failed_runs":11,"raw_occurrences":12,"matched_occurrences":3,"runs_affected":2},"rows":[{"environment":"int","failure_pattern_id":"a52d3f63256add31d93e084d23258696e3ce9f667510ec06dc509046b07f2f14","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD","STG"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000df6cd8\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-pfttlc9b26s6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\",\n      \"name\": \"ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T22:57:32.075098968Z\",\n      \"endTime\": \"2026-09-16T23:16:32.974740274Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000df6cd8\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-pfttlc9b26s6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\",\n      \"name\": \"ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T22:57:32.075098968Z\",\n      \"endTime\": \"2026-09-16T23:16:32.974740274Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100351631508901888","occurred_at":"2026-09-16T22:30:19Z","signature_id":"f7210bc017800c8d3ce9292e2b300557e56751eedb6338b78091f529330d0435","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"44813a24efc9a4e87de23a1d03f218de3f44f990811139bedf70528921d1e908","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:162]: failed to verify HCP cluster ephemeral-disk is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017886a8\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:162]: failed to verify HCP cluster ephemeral-disk is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017886a8\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100532556209328128","occurred_at":"2026-09-17T10:29:22Z","signature_id":"a7b733023a299f40faa1bcb49b8265ed28d3f85ddfbf335757885b57a1e7d20f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"221cb0b46f3bef91cff2ee06c068eca8fb3646ef7eeb7ed9cd39ae59c1446e81","failure_pattern":"list nodes (nodePool=\u003cnodepool\u003e): Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"list nodes (nodePool=npdg-4-21): Get \"https://api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net:443/api/v1/nodes\": dial tcp: lookup api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:59019-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:612]: failed to get node release images before downgrade\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0019a0220\u003e: \n    list nodes (nodePool=npdg-4-21): Get \"https://api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net:443/api/v1/nodes\": dial tcp: lookup api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:59019-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:612]: failed to get node release images before downgrade\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0019a0220\u003e: \n    list nodes (nodePool=npdg-4-21): Get \"https://api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net:443/api/v1/nodes\": dial tcp: lookup api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:59019-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100532556209328128","occurred_at":"2026-09-17T10:29:22Z","signature_id":"06051e7fd69c31d191475d70ad084a2f5f338204d9dcb2ccb5a0566c0fc34ac7","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"prod","summary":{"total_runs":37,"failed_runs":30,"raw_occurrences":142,"matched_occurrences":131,"runs_affected":19},"rows":[{"environment":"prod","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":51,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7027027027027026,"anchor_occurrences":51,"after_last_push_count":51,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,51,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-n6q68g with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000be8cc0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-n6q68g: failed waiting for cluster=\"np-version-upgrade-cluster-n6q68g\" in resourcegroup=\"rg-np-version-upgrade-n6q68g-z5wvsgjdr7l9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n      \"name\": \"47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T19:55:24.695867176Z\",\n      \"endTime\": \"2026-09-18T20:24:27.02816156Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-n6q68g with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000be8cc0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-n6q68g: failed waiting for cluster=\"np-version-upgrade-cluster-n6q68g\" in resourcegroup=\"rg-np-version-upgrade-n6q68g-z5wvsgjdr7l9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n      \"name\": \"47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T19:55:24.695867176Z\",\n      \"endTime\": \"2026-09-18T20:24:27.02816156Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de50e0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-bddwvtnwz8x8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n      \"name\": \"1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T19:54:45.082839017Z\",\n      \"endTime\": \"2026-09-18T20:23:47.024709747Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de50e0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-bddwvtnwz8x8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n      \"name\": \"1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T19:54:45.082839017Z\",\n      \"endTime\": \"2026-09-18T20:23:47.024709747Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010085a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-266qhn49mfp5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n      \"name\": \"724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T18:06:21.25552087Z\",\n      \"endTime\": \"2026-09-18T18:35:23.923631594Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010085a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-266qhn49mfp5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n      \"name\": \"724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T18:06:21.25552087Z\",\n      \"endTime\": \"2026-09-18T18:35:23.923631594Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"020177279fd83a306292e61e3b6eb559ce1bd5ba1e079600d6fe28da054a781b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"08737fbd1b615be947880829b89bc4c3a56995f666d4512eff62626a308dd556","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"087e1145946b4175b23e226d2c422f8fbe6adcface041612342c65176f135291","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"12c88ab455c66941eb056387e08ab482c4399c448c1844b1db1d64ca2a21923b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"18459d5e026c91320fea1e6f4619d65e0f3c2cce677a7bf1890dbe9529aa82dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"18cde222ab56ffb759e053d76ee6e19c739c516c2db2d311138d345c27569034","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"19596332cc8edb7c2fe7ff05b62421eeeea53668cb594d6f172e695930cfc93e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"23eff0283d14160643a2c1b951b106f62c524eb771d25036e9a90406a0924c53","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"279014bba3c91c8c1147706aa208a7ca8dab431b40a05ae4dc2507ffe4bd82d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"2a4e0071788107d1b0f28cafcae777b74d59c2ed92ca22e3caeb7564b1685be1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"2bd526e04864e297297108f31c386e629877109b67a0cd5577f66e966f4820a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"2c5047fbc800377676e3159b01faa4b14e97abaaef72bb1d0f7c177a6a8e963a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"2e5039616cb1c8d7f4ac14c50b2a06f1fe40081131c5acd543f28d2cc514abec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"300283533309312abfaf8eb06c5bc243249dba952ac99795631a8cca96c1b04d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"3a36b2cb0e45db6b1b5adf537bf6a65a0dc2daa7ebc74fcf104789a0636dc715","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"4d196dbdd6adc5cbd0b42174989b35c61699ec802a9b43e61050ca43a4a526db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"55df98f2e805e8d535825529f230ec77e5caf64b117f366392092a2ef0f04238","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"55f11975675616706103b6b7daff6a4b423e0747ce59c5e7da66f3dc349981e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"596b0ffa1cf24f21e4c89da0dd1bf8032daa39c9dfe890cbc997391f5e16d758","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"5bd085aa7d4e7efc364c83356bec393db7595ff945fba164d18f342fedc7d074","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"612c1b0bfa8c63e918534f55b46917bda5292bb668f43f5109397135c5f16839","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"7d0f6a2cda1c21acb4480d38818269f535fbffb2494dac6f98b849200db6bc5d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"807a30302c1afe55628a0e91c1d8d2d1cf846d02300aeafd38841fdc5629959a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"88f393269cef98e4be13d3c8ab69c7daf8948bb76a2277ef306df923eea104ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"8c17a49c3b1087a5252885b16d63b8887ba96a24c838b70dae74c83a59595562","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"8e3503c7dac56f3fd400f32d8b3f5b71ecf5acd0c55bc2d5e52b0b130348d093","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"8f8a4a3e3e075b660c1fb0534694eabcbec0ee97ddf1c7de50ae027a7c989e5e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"9253c3493335b5dfd398622bbe3451ad8bddf74417d1a8c0037aece178838cde","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"980439aafe10d6730454288b03e5c4b325ad4d9a23209ccce9ea0925ca8727b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"99a0fc3be3299d125a1312f83223671d90940dffaa62fee2e5b18166cfca67f4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"9da2d69669521624ba370e29cdccd478000fd77fa5acf531ed0100da09e598f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"a086d8d16ebbda1f55085340c84766813ece581729f4e4db8f61f9c0b1339c8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"aba4ff8b5edea60411faee257f0fcca1514022f58c987167a980974659ac599f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"ac027a2fae7f9c119c5d13e433aca363beeb537c7f13236bce5b4509168d72e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"b09c9c83b49154876ea9edaee019cac3ced0de0e27ddbe6889ee50f6b423f2ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"b971663cd5f630a57997cfad6d7efdb8293b9c2c40b6d6ab7621acde7a5c5511","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"bec4f85eb91db9f106f1504bbcfbd5d619f318abcff9dccca5f00cd3af48e378","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"bfbf541997c9dcf2457e39f6d9a3f3675b64df774c7c81e642e24bee8b14665a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"c71f8c34107674e9b83007fd4c4382acbf933644d3d70e04b10c2bce3d213437","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"c9577e8c1b375d41d27d0e18df18642cc9e2a302455c27112af6cdf04b27da87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"cb28d0b51212f011a6a9a6a3530556dc58c4a4d691d64d362e422cfc091e42ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"ccebff601285f1ea3ac04ae7a355282a8c0e14e4f8eb582eebc840c11f6d9a08","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"cf464e2390b41d88251c211ae76f31739afd822d6d94414c2b9fbfc2f709feaa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"d992a344dec95549926180b213136a61987bad3b2b2cfe5a8cedabfb0cba64d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"d9b87f83858caf3b228ef9b132bcd625c28601b6944272225765d7e4212bfbd4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"eabc0e9f060f2e7a3e41ea442762aca89d3441bef1655db8dca9669defa6ff8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"ec2f8559597c2e2e96353159ad3c18cb0e7f4c1ae2d3f60c226c0178a7ef5d2b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"ed3aaafec71e925920ce96600e6accd52368d8c067bd7c98fb0f3dd54c8990b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"f52d9598c009df2683fa8e80fa13ffac13c236506f242fa3de8742e288cc9518","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"f6a9a5ad677f486ebe2e2116b48d59eff2dab4dc02f9785191be923be7559549","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"fcaef2f74196ab4f30c8d92bee5653f6add69a944685179ec76976f1aebee54b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":24,"runs_affected":5,"failed_runs":5,"run_impact_percent":13.513513513513514,"anchor_occurrences":24,"after_last_push_count":24,"also_in":["STG"],"trend_counts":[0,0,0,5,4,5,5,5],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001896cf0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-79mxdqgdx9t6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001896cf0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-79mxdqgdx9t6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00286bb60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-r4zrv9dgbv7h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00286bb60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-r4zrv9dgbv7h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0016fa690\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-gc2rhgk55bsk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0016fa690\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-gc2rhgk55bsk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","signature_id":"17a023c6a46245432cc416ef8d62649f55ccb9fa39f11d5aea7cbfa4336d9de3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","signature_id":"328adc8fb8c132a962c8f7e8e644e383234b1bc202be2521e5dea0f4d52718f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","signature_id":"609d9384e69628bf55c86d248dad68fa3cfbe04261dd0ff9804138d57638cb07","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","signature_id":"c1035688eca499c52db3ca5bb0c7a77093840ac45b328b2854d0de497f1f82bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","signature_id":"d1a6fefdadb947de8c33d9842ac16f2fca17157a7f575a18d904be8040d389a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101446375928500224","occurred_at":"2026-09-19T23:00:33Z","signature_id":"a358e3006a61fa65263eb299d48cfa24f3c2010b7624b474fac160ad556540aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101446375928500224","occurred_at":"2026-09-19T23:00:33Z","signature_id":"a6ea02a444e3281968f614a4adbbdbb4e9a9f89b627c67726092730042c3145f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101446375928500224","occurred_at":"2026-09-19T23:00:33Z","signature_id":"b6cc96b558f0d49dfbe6b627f41acd67faac6147bbf1b1eedc3e7cf6be69edbe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101446375928500224","occurred_at":"2026-09-19T23:00:33Z","signature_id":"e660b31fcd691c2df8d8cde9eb3a53049a3a8d6c1455b1472e30c47f47c36258","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101446375928500224","occurred_at":"2026-09-19T23:00:33Z","signature_id":"e872f27bf86ae757c1610dc7c7f82d9a07bd8020b8f154abae9c7ff458b5ec57","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4e7909db88252003612c7eb675142841e8737860b4601bc7ba1c1c1147d59585","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"6b756bbe3882dd3ad1f64906dbb7701108fcd8092065edc0fdfc339e155785c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"9ef798cef3d572a0e66143f21817eeeceae70fa379c7c12f948b17be196688fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"a1f4334212b041c83192bb8549659dd10bbe7f9e8bd239e3cb44f42460834cd6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"d656f1256e1df9e299a9ee99092b293d1d1b5f5763759abb016ae0c94e36cbee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"19d9edd34f8e48701cd8bbda924191e80690cc1893c6e47aa7458abc045a4095","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"ad4219e21e3f54d13de0b5a81134597e91fe66a1442cf0f6968126d652406ccf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"b64e2e209238a4291f7bc6ca875c4002b2b72ce61c22b00cc341eff1d3d7d329","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"d144fcda6ced4631fcc19c7abd504ffe793c4fffbba643dd7b8a0d80f55c67e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"494d690c10401d515d7d97bb1b43282cd5512bedf4a8cf0e93991a312d0f71e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"5f0d2310ed06414a49e9f1ddd637c2a2516af067a00a8168124b28d160b1f9b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"a91c26d043927cb825f419af86a21ed7c54daa9e6da09ecdbe0fab7af0a8ab85","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"b2ee8d4e7b429ad69336ec70ed0fafeb7a58d728ccf645b31d75121fd78def37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"e2f2d3555780f9a4be9e4c49fe522f13754292efbc3c1299543a5305777d3351","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":16,"runs_affected":4,"failed_runs":4,"run_impact_percent":10.81081081081081,"anchor_occurrences":16,"after_last_push_count":16,"also_in":["STG"],"trend_counts":[0,0,0,0,4,4,4,4],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d12020\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7hkgvb\" in resourcegroup=\"rg-np-version-upgrade-7hkgvb-db7f9x6gvf59\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7hkgvb-db7f9x6gvf59/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7hkgvb/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d12020\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7hkgvb\" in resourcegroup=\"rg-np-version-upgrade-7hkgvb-db7f9x6gvf59\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7hkgvb-db7f9x6gvf59/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7hkgvb/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-dr4c56\" for cluster \"cluster-nightly-4-20-dr4c56\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d101a0\u003e: \n    failed to create NodePool np-dr4c56: failed starting nodepool creation \"np-dr4c56\" for cluster \"cluster-nightly-4-20-dr4c56\" in resourcegroup=\"rg-nightly-4-20-xfvxqd785djj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-xfvxqd785djj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-dr4c56/nodePools/np-dr4c56\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-dr4c56\" for cluster \"cluster-nightly-4-20-dr4c56\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d101a0\u003e: \n    failed to create NodePool np-dr4c56: failed starting nodepool creation \"np-dr4c56\" for cluster \"cluster-nightly-4-20-dr4c56\" in resourcegroup=\"rg-nightly-4-20-xfvxqd785djj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-xfvxqd785djj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-dr4c56/nodePools/np-dr4c56\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001070ec0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-zrpt5w\" resourcegroup=\"rg-np-dg-minor-zrpt5w-4m5b8rktsw6x\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-zrpt5w-4m5b8rktsw6x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-zrpt5w/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001070ec0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-zrpt5w\" resourcegroup=\"rg-np-dg-minor-zrpt5w-4m5b8rktsw6x\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-zrpt5w-4m5b8rktsw6x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-zrpt5w/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","signature_id":"0230eab10a5b6baf7463f9373d53a9a0d5e973ea498e4480c157f3188984c7e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","signature_id":"0c0591f9586da1ceac9fcc46f897431c4bbba6e1aebadce62e31ef160dc87f07","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","signature_id":"4e8137a2ef56b5461da5789f436cda8495e7a5d9d716cbfd68569ebc575a8b1c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","signature_id":"6196ac4300552dfdb8e74a1b49f7d8225f6e95469be8ee12fdcb6a58f8ef5c17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101446375928500224","occurred_at":"2026-09-19T23:00:33Z","signature_id":"17dc4c1337f5c0709c186f76967ee7ffb18bab887b6a0c2505b44db1a97933f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101446375928500224","occurred_at":"2026-09-19T23:00:33Z","signature_id":"359e89867e4d6a12b6e211ff92d9f2d97ba9e31df1109aa240d3a17f8c7db4fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101446375928500224","occurred_at":"2026-09-19T23:00:33Z","signature_id":"490f4a7c3c2ebd6966eb03cd9cb45259accf8e98ff9ab8117af7c08975a0d6ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101446375928500224","occurred_at":"2026-09-19T23:00:33Z","signature_id":"96a43341df89eae84cc844ec056958584f9b7ba72133b4a9b37f2255b7a81217","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"491628b312668f06a0a9bf2464e7de82cea05fd0fafcb43e52e269cbc904606b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"8807298fe1e8269d8ba1826e326fe8888ecc37b86d4680f87d9ef650960b1b45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"889f4b2cac9cd859f4a4017cf24ccfb20aa30ab6ad074847bdec286491246277","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"b07971657c34858950043966384bf3450b2bd8793b71995b336ae5e99ce5d756","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"475d18438d6129ee057302e14cb4929cac287683cc42b505124692ed9000396d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"5b8b71dd55429c7ea7e12ce3e8d437a79ae271abc6b3fd1cf66b2d542a2f54ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"9f76555451648e830890a7c9cd43251ffcd8ea938c9675715ab08891b03af525","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"ae583f4fcd37f743d2e4acb91167ea667e50e9c13e01db10c235ffe6dd99caa7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":15,"runs_affected":9,"failed_runs":9,"run_impact_percent":24.324324324324323,"anchor_occurrences":15,"after_last_push_count":15,"also_in":["DEV","INT","STG"],"trend_counts":[0,0,0,1,5,5,1,3],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":9,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000ed0f48\u003e: \n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-75p2xk77rtzr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/65cff58d-536e-4d46-b35b-58f67af72b80\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/65cff58d-536e-4d46-b35b-58f67af72b80\",\n      \"name\": \"65cff58d-536e-4d46-b35b-58f67af72b80\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-20T02:16:51.798119339Z\",\n      \"endTime\": \"2026-09-20T02:35:59.041922136Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000ed0f48\u003e: \n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-75p2xk77rtzr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/65cff58d-536e-4d46-b35b-58f67af72b80\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/65cff58d-536e-4d46-b35b-58f67af72b80\",\n      \"name\": \"65cff58d-536e-4d46-b35b-58f67af72b80\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-20T02:16:51.798119339Z\",\n      \"endTime\": \"2026-09-20T02:35:59.041922136Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00118b8a0\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-7sn2sc\" in resourcegroup=\"rg-np-skip-minor-7sn2sc-x6d7knwq47tj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0555a946-aa1d-468d-a6ad-48364e5f992e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0555a946-aa1d-468d-a6ad-48364e5f992e\",\n      \"name\": \"0555a946-aa1d-468d-a6ad-48364e5f992e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-20T03:23:26.91112939Z\",\n      \"endTime\": \"2026-09-20T03:42:27.966002583Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00118b8a0\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-7sn2sc\" in resourcegroup=\"rg-np-skip-minor-7sn2sc-x6d7knwq47tj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0555a946-aa1d-468d-a6ad-48364e5f992e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0555a946-aa1d-468d-a6ad-48364e5f992e\",\n      \"name\": \"0555a946-aa1d-468d-a6ad-48364e5f992e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-20T03:23:26.91112939Z\",\n      \"endTime\": \"2026-09-20T03:42:27.966002583Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00011a240\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-h72vxbzhd2w4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5017a36-2ffb-4b1f-9f5e-28e6a6204319\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5017a36-2ffb-4b1f-9f5e-28e6a6204319\",\n      \"name\": \"b5017a36-2ffb-4b1f-9f5e-28e6a6204319\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-20T03:10:04.541118917Z\",\n      \"endTime\": \"2026-09-20T03:29:07.643791064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00011a240\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-h72vxbzhd2w4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5017a36-2ffb-4b1f-9f5e-28e6a6204319\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5017a36-2ffb-4b1f-9f5e-28e6a6204319\",\n      \"name\": \"b5017a36-2ffb-4b1f-9f5e-28e6a6204319\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-20T03:10:04.541118917Z\",\n      \"endTime\": \"2026-09-20T03:29:07.643791064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2101491692631756800","occurred_at":"2026-09-20T02:00:37Z","signature_id":"7dea51084c53cfca68d054fdb2b8ed1e7adcbbca9101d85ead03c9a16e317eb4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2101491692631756800","occurred_at":"2026-09-20T02:00:37Z","signature_id":"8381cc68ee86757495de46236ed3b123ff2a372a4b908f8548b0bdace5e822bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2101491692631756800","occurred_at":"2026-09-20T02:00:37Z","signature_id":"933e30630642a92f4e79135b5aa54df21013442019274bf9c20132e6dffe4ecc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101446375928500224","occurred_at":"2026-09-19T23:00:33Z","signature_id":"cf44656c4e56af1df483634667436799c7ea9fd011485f1ad0ab3c202890dfa8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100826864959361024","occurred_at":"2026-09-18T05:58:50Z","signature_id":"7cb7501ef4b819fdc17f5b5e3ca2619d486971f5a3c25a5533b8ff273a412172","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","signature_id":"60cc795ca9532d84d0a6a8c864d8b05a2f5285c1bfd6c8aa612a88d8890d8bb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","signature_id":"6e4745afece375c35c57231f904892210b15b74988a6f2baceda810fa43da8e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","signature_id":"8be7c23a1e56f277787ec21fa08eb46e46af4b8580d42385a8b71372839091d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100762866352132096","occurred_at":"2026-09-18T01:44:31Z","signature_id":"812ff7352462ae5b2ffc2a98177a772e72a7a199983bda3af2609cfd08e8cb77","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"8af91b6a912b93a55a51953f18f51b6e88bc18db738006d3b71d9e5d820486c4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100694048263114752","occurred_at":"2026-09-17T21:11:04Z","signature_id":"be41217a9374881e99f1a9416ccadf32d5a5da3fe931b3a6439f304be981164d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100694048263114752","occurred_at":"2026-09-17T21:11:04Z","signature_id":"ce783312029a03f3c6f81cf899cc96d5f27f953623a9a5fa1b6b579e78455f50","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100694048263114752","occurred_at":"2026-09-17T21:11:04Z","signature_id":"e9d197f55b7c8094b6f94fd2ac343ddfe7d54231e67ed0c0282b273041aea3b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100404440946184192","occurred_at":"2026-09-17T02:00:16Z","signature_id":"ce1e7db080deea06cb7610e1f4b4bf939ebd4afb01e8c1a7763c9d9cd1a9bd32","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"aae2457e0f58577b233ddb5159361da1324cc74ad01ffe504591fd6f9131c0dd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","failure_pattern":"ERROR CODE: SubscriptionNotRegistered; provider Microsoft.Insights","search_query":"ERROR CODE: SubscriptionNotRegistered","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":16.216216216216218,"anchor_occurrences":6,"after_last_push_count":6,"trend_counts":[0,0,0,0,1,5,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a45890\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: SubscriptionNotRegistered\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"SubscriptionNotRegistered\",\n      \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a45890\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: SubscriptionNotRegistered\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"SubscriptionNotRegistered\",\n      \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00117a2a0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: SubscriptionNotRegistered\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"SubscriptionNotRegistered\",\n      \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00117a2a0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: SubscriptionNotRegistered\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"SubscriptionNotRegistered\",\n      \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00088e570\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: SubscriptionNotRegistered\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"SubscriptionNotRegistered\",\n      \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00088e570\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: SubscriptionNotRegistered\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"SubscriptionNotRegistered\",\n      \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100894117604102144","occurred_at":"2026-09-18T10:26:04Z","signature_id":"3269dcb6be86d6a6047ecae085ca193c45aec85e3774519113b1593c115dfa87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100892735677403136","occurred_at":"2026-09-18T10:20:35Z","signature_id":"d7f2aa1ec90b6da0508d97952c26f5e68a44fec6becea93e11a0c8acd2ecf80a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100890223054426112","occurred_at":"2026-09-18T10:10:35Z","signature_id":"5533b31e4276757a4afc74d1ad67148e45245ccef8ea260d65b9e08077693a1d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","signature_id":"6fd32498c414e944f0c010bd303fd297e25f438aa938164d7b89331e4fe1bf3e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100762866352132096","occurred_at":"2026-09-18T01:44:31Z","signature_id":"f164ce20210c5901433b68d8b32c7bafd9db9fbdb4b9f088ddfe4a53eb8ff36e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"a0e7d58faa2918f496570ed9c5f40cc4898c7cc2d361ef4097b193762f74fdcc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":13.513513513513514,"anchor_occurrences":5,"after_last_push_count":5,"also_in":["STG"],"trend_counts":[0,0,0,1,1,1,1,1],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009e7440\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-mtt578qwjxhd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-mtt578qwjxhd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009e7440\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-mtt578qwjxhd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-mtt578qwjxhd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106b280\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-mbf8fbn585g6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-mbf8fbn585g6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106b280\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-mbf8fbn585g6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-mbf8fbn585g6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00298e7c0\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-9f9tbz46mct7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-9f9tbz46mct7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00298e7c0\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-9f9tbz46mct7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-9f9tbz46mct7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","signature_id":"e1e9f91cb19890a0eb0ac4baf7b91616569638fb83a8ccd96e69e04addc7141a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101446375928500224","occurred_at":"2026-09-19T23:00:33Z","signature_id":"ec3d32d5df0b075d997dd85e88314e4c16b2e869d3604bc889c5c8f9ad43f462","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"19958c8a5897194ef3182c6ab16be4ac9029906aade55bb4256e8a9bbd189d3c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"eedd7ed28a2e498144c6499476b15f9b62b521f665937a7144186b5981a0ab86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"70af6c4c5a716b4a2eb8db0539e648de82b2496a872a99a8c6a98074fd41d4c3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":13.513513513513514,"anchor_occurrences":5,"after_last_push_count":5,"also_in":["STG"],"trend_counts":[0,0,0,1,1,1,1,1],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0018004e0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0018004e0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000ac15d8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000ac15d8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00055c870\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00055c870\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","signature_id":"cbcfc6c0ea890936fe78377d8bd883d847e8c26fd22ca0042a2db994fd99152c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101446375928500224","occurred_at":"2026-09-19T23:00:33Z","signature_id":"be4fc63136d0db596d5249b63a1ee31147309b0edd4d8e30faa9dbc686209828","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"31f227f2999d078a72461d40e7b0f7d39a61c55e78688c72750e61796cea0c0e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"787555a75756b0bcd3c50e5022f58b6e6c990b64d6f87d9979ae97767e7916f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"f5ce501d2dd69dcdea5a25282314ce7ff4dede1a06ceeb918b7e36499f9665b6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"e3e491019ecf3f140e87c882b7d45ae9d3071ed1df24cc639d33c7a9d0e29087","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.405405405405405,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,2,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014ced60\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-wvt6tvvldjdg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n      \"name\": \"80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T18:06:15.268087105Z\",\n      \"endTime\": \"2026-09-18T18:35:15.027123676Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014ced60\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-wvt6tvvldjdg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n      \"name\": \"80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T18:06:15.268087105Z\",\n      \"endTime\": \"2026-09-18T18:35:15.027123676Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000860220\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9vjp5gmczk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n      \"name\": \"df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T02:08:07.344249043Z\",\n      \"endTime\": \"2026-09-18T02:37:10.01628355Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000860220\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9vjp5gmczk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n      \"name\": \"df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T02:08:07.344249043Z\",\n      \"endTime\": \"2026-09-18T02:37:10.01628355Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003701249380352","occurred_at":"2026-09-18T17:06:01Z","signature_id":"e22784c3104fac10937a42e7b5069677dd3f893046b62e795e223bfab93b9fcd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100762866352132096","occurred_at":"2026-09-18T01:44:31Z","signature_id":"736f2b0814fabfc59b7e462d9755a0e748e4d396c2743f81540491fd582dab16","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7027027027027026,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d3135a9098d386ee7983deae617b720e2fc6a509bbc3da23084b1fee26bf425b","failure_pattern":"ERROR CODE: InternalServerError; detail code OperationNotAllowed; detail message Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-sgbzk-\u003cid\u003e' since another operation is in progress.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7027027027027026,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00160b7d0\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rzlnqd88ksbr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0a92b991-1627-4a97-9fea-a9edfc1c8c70\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0a92b991-1627-4a97-9fea-a9edfc1c8c70\",\n      \"name\": \"0a92b991-1627-4a97-9fea-a9edfc1c8c70\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-19T02:54:29.386299179Z\",\n      \"endTime\": \"2026-09-19T03:13:30.082275878Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    encountered errors downloading console logs:\n      - VM \"cilium-cluster-cilium-np-sgbzk-2tmm2\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-sgbzk-2tmm2\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-rzlnqd88ksbr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-sgbzk-2tmm2/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: OperationNotAllowed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"OperationNotAllowed\",\n        \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-sgbzk-2tmm2' since another operation is in progress.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00160b7d0\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rzlnqd88ksbr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0a92b991-1627-4a97-9fea-a9edfc1c8c70\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0a92b991-1627-4a97-9fea-a9edfc1c8c70\",\n      \"name\": \"0a92b991-1627-4a97-9fea-a9edfc1c8c70\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-19T02:54:29.386299179Z\",\n      \"endTime\": \"2026-09-19T03:13:30.082275878Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    encountered errors downloading console logs:\n      - VM \"cilium-cluster-cilium-np-sgbzk-2tmm2\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-sgbzk-2tmm2\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-rzlnqd88ksbr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-sgbzk-2tmm2/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: OperationNotAllowed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"OperationNotAllowed\",\n        \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-sgbzk-2tmm2' since another operation is in progress.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101135291371753472","occurred_at":"2026-09-19T02:24:25Z","signature_id":"e69dd1e536769a78eb7dfcdcfc7a9d0f9f7cd9249f2f7c9f87d1bd8d4faf8d20","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7027027027027026,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b57f3261aa83aadd9ceb7988b00830acc29894d12dc4f5f19baa862276967565","failure_pattern":"cluster operators not available: console (no Available condition), dns (Available=False, Progressing=True, Degraded=True, Upgradeable=True), image-registry (Available=False, Progressing=True, Degraded=True), ingress (Available=False, Progressing=True","search_query":"cluster operators not available: console (no Available condition), dns (Available=False, Progressing=True, Degraded=True, Upgradeable=True), image-registry (Available=False, Progressing=True, Degraded=True), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False), monitoring (no Available condition), openshift-samples (no Available condition), storage (Degraded=False, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown)","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7027027027027026,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc002a4e018\u003e: \n    context deadline exceeded\n    cluster operators not available: console (no Available condition), dns (Available=False, Progressing=True, Degraded=True, Upgradeable=True), image-registry (Available=False, Progressing=True, Degraded=True), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False), monitoring (no Available condition), openshift-samples (no Available condition), storage (Degraded=False, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc002a4e018\u003e: \n    context deadline exceeded\n    cluster operators not available: console (no Available condition), dns (Available=False, Progressing=True, Degraded=True, Upgradeable=True), image-registry (Available=False, Progressing=True, Degraded=True), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False), monitoring (no Available condition), openshift-samples (no Available condition), storage (Degraded=False, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100173746357669888","occurred_at":"2026-09-16T10:43:35Z","signature_id":"cfd34eb004cc5a7cf7577f79d571b368c1d8dfc53041aa652f9465c361a3b4a0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8j5t9t6fn8sk\" to finish creating: GET https://management","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7027027027027026,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc00381c7e0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8j5t9t6fn8sk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d87907d8-e7ff-4012-ba96-a28fad371f71\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d87907d8-e7ff-4012-ba96-a28fad371f71\",\n      \"name\": \"d87907d8-e7ff-4012-ba96-a28fad371f71\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-19T03:13:36.546108406Z\",\n      \"endTime\": \"2026-09-19T03:32:43.229867603Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc00381c7e0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8j5t9t6fn8sk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d87907d8-e7ff-4012-ba96-a28fad371f71\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d87907d8-e7ff-4012-ba96-a28fad371f71\",\n      \"name\": \"d87907d8-e7ff-4012-ba96-a28fad371f71\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-19T03:13:36.546108406Z\",\n      \"endTime\": \"2026-09-19T03:32:43.229867603Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2101129282930610176","occurred_at":"2026-09-19T02:00:32Z","signature_id":"593c264f9fc5ba4dca77e82b05131159c28cf9111aa6d4894a517c49a66d67ae","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-tzdswnvvvrrb\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7027027027027026,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001376700\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-tzdswnvvvrrb\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001376700\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-tzdswnvvvrrb\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"e69d26f721f1d960b9d056d864133ef61ba83a27918de693c75cfc743d5fc4a8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ql4cwwsm948w\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-ql4cwwsm948w, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7027027027027026,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010a86c0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ql4cwwsm948w\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-ql4cwwsm948w, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010a86c0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ql4cwwsm948w\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-ql4cwwsm948w, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"60ea4497c6186b3e803d7e6ac044255c8c915fd98b1ff264e2058f0d46b0df0e","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":20,"failed_runs":19,"raw_occurrences":296,"matched_occurrences":286,"runs_affected":9},"rows":[{"environment":"stg","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":268,"runs_affected":5,"failed_runs":5,"run_impact_percent":25,"anchor_occurrences":268,"after_last_push_count":268,"trend_counts":[0,0,0,0,0,52,108,108],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa0100\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-htkcwk2h26jm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-htkcwk2h26jm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa0100\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-htkcwk2h26jm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-htkcwk2h26jm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00111e180\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-svx7g8lfp7vc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-svx7g8lfp7vc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00111e180\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-svx7g8lfp7vc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-svx7g8lfp7vc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009903c0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-h5k2wts9ndxl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-h5k2wts9ndxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009903c0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-h5k2wts9ndxl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-h5k2wts9ndxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"0035badf1d52e79b46a89636294ab5175c1764ce57ed472580f51c38a17646d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"051ffac544e878c8e8e19b0de538556612bc6ea491ec662a5d5e352d905c7b58","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"0726e178dba9603bcc13b5275955ebffef79669ebe41f4359f6650518623f84c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"0ce1d88e3db733abc7156b9265acbfe26718a282fc729d00d437842a310dcd8a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"1b89ced61ee5be756db27ad1f65de34e90424823131553ce990f5e8bfac77fdd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"1cd687ef7bbbb14f8d590c7438121eeea9ec4020c01b7de0e34917b9f0992ab8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"207cda487ed44811f5480f8f37446d9601422530a9a36879efe9f029372d698c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"2121c5292a8ded95a05fe5474e5fc5ef3ac3f89d37ac3669aef473ed2c3b43da","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"21d22b8dae517e3e2b72ee69216b49b53e62a80fc88bd50de4149218570a2168","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"2262ff705c982dcfb0f913112870bd6e548d66fa181c0b45a3eaaaa1d46212fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"330777c600b2c414fb1e06afc890222d3aa747a1bfc6c96b43ff07b15fd45d2b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"35fca88da7d43a424bd831003af76469f024f27d52d9191f8d38f4ebd3598aab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"3deeda3dacfcc6e9613866522eff1df6ffa855f675bfcd2b3400d1a8bb0043e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"3fcbeffbfcb7999ced44c9ba14ff662338696cdba1dd77ab618e21b94c73c35e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"431ac06c4759e7d928f509a50e2760ae01dbe5db17da9c2a8bfbf5a018a1755a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"446e04f4f7e6b9c7f65a0613b14639d4c40ffdf6cb08a3fc64d946bb72b161fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"44f5ce1f36b75dc92b37eb2219e5536cbf8dae57ad133af027cb383345b0f81f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"463acd71cbfe6a54942c3369058385b4a3790957af355b5aee86ed77a281a952","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"4b692397a309ba5941c3b37f467df2509c51aa2665cd1ded857f0c6b44b0037c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"4de4857ae4514f966ab0b2b4d8690d29ba2156a7280cfb00cab15b74f8ab103e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"51cdc90b4d51d3970f1db21a4d2ae21e451183c4709e4ff89759bb06ce3c0d56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"53503846cd73044f90ce87f6819d59a29a0c5aeb170996d62d78a12b6c18d33f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"54559b83068e8ae8910f708b852aa10a1bfe8f38a82051dac25e3b3c2c041dbc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"5c49ce9d1f1cb82ca579db2f9e60af5804350931bbde5c891be88d49b03e87af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"737be13fabbdd7d8938f70a23dfc8dc615ab58be6134cddbe08c7cef4614840d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"8398a4952142c55a945498132d985a4eda864ceb95312242d7edc8241b8c3c26","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"8b287cd672b9ce5c760ea23a53dddea3b4f9f998a16fc2807bcd881b62d12fa8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"8e693b389bf5da5107e36661d6ad4dccb8934c130010b5e39c954a1624c42673","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"90a8e110e2d0e56022780623aca6ac1efad98929e39705d71e89be66fdd48f3f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"910729e19a1a95fb2ecb26f059eb401491443d77afb01693df52c373ec5864aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"9703f084cb7b88b8745c68c7057a0dee5319018ab6983f426787450090e42982","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"9934fed1fdd8af1db46791b1dfefd62cd08c923399e036e187caa7b300776d49","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"a40e7b1a5607f31536cb5cd9f68551075682d33d966e6099cae407be5eb884db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"b1bb5742be13056dd2e18f31ca353f24f129705448e865edc5ca8d30ce5acf58","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"b3bf8aee173cbf7a075ad4f0093992fdf2f9d3fdbdd4c79ed228a07fc41be6a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"b5e7a08dd6c3cbae26446e50c6c7706b0f70b5b0efc3ca7ffd303717bf777b08","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"be9743a3b7525b79eee4c11217ab516d2317647c36b86a981d9439f483ce73e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"bf9220ee57546bfb98ca09c23ce1bcafa0fc9b3ad8c908396fb22e88d8e4390f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"ca0e44d54763e94dfd93a8e5d50e13a45230df631e95060b93c9af16138398d7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"cb9f12b0fb85348fc974d4c7f92ba92c9dc3e09d94ab423572856292667ab5ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"cc29f32f78f823cb461df160d8a795ab0f7b89379767e103cd258870d5376e6b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"d071dca948f099738552907e3ac446399822a96c75018da19fa4fb514960b791","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"d7f52fda9e0a37e1b44a584e50dea99d5f305e1aec140de81d09c8899f6ecd59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"d97958181598a8ed71cf272755962a1a133abe4c9b76728f8a3433630aa69eeb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"e1f2463dbdd365ac2c40122a7bd57873d9d3da823de699c668c0f0b4da3bf256","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"e6b78af356090863123d3e08d8801c8ba0ac603f871ed2cb326d9ca914c11f0d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"ec1323a0e04d40f520c159983f60e8cb338c88ccf1ebf099cb4705b2023d96fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"ed63787da4d3fd4cb31e6acfb7525e56bc4189b4c555b5dee49e05c0357ee203","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"f036046fe003926a26f5c7f66f4226f6e0b880c29bba5601c195ecc014eafa48","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"f4595528f2155d256fb2bac014a8f2cdd20e8e748731ae1d9a50be16b84426f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"f54684b919b3a17e9b0d89e05f999162d7714a3aec1ee3f6660f91a081bd3ae2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","signature_id":"fc16bebb53c8d6af1459f162bc0d57eea7307426a6d7301cf07495177dd5b525","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"041f672a3e18a2b28c596e30271d2c9aa88c28c3ee7aa0cbb42b72930798fee2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"0bc9ed5ddde6d1c41f63cfc989be8d7ca185c5f36de781135291034f103b12b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"0fe0f255a1fd9565d5b7035ac907cfe87a46d541f77f6bab233e15a30fe08e4e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"112fe54e3a4ac97241812ae495e19957375dd2aedc5ee183bdabbc212cb22b14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"11f627ea2f83c9553f9eae34198a8c69b4a402111e6e9f67c4c5ae2ec2a9f2a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"151bc8a39c568a5acc1d67a76acc23403f08945c62637d077e8031bb8940e682","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"1b55e9418c26b0d134153b6a2d57fae30390b593c966c5993d9823c107365902","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"1daa02e05dbdc0b2d1cf1d494fe0760919fe977375ff26fe7b7ceb2569b3ff41","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"1e49ee1e36e7e1cd42104eac5bdf587b01098b06930d85e82c33036cb9279e20","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"2009e10126e3e1bd18661ed5139d7b2050395ed3a6f9c56bdb146b87a1468837","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"22aefaad04ec58d0bb6b547b1a1b3ad95ec0618155b7a855fb859bb3539afdcf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"22f4a53d1fb05825ca9b8ce928e01759b6a5fe1374579d62f59affa851a7cf43","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"2ab60864406702cac34dd6b5ff1ca75ea58460350850b988fc3ec8976ba1dc6d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"2cd4eef6dce6c922a8e75708f23338ab8e87f52778806c126f46b6e66a6fa757","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"2e3fe351111002c119c19d9dea7ccffdf52c6ecce3468c097eb0df92ee24040c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"3c11fb560d81575cd3e1019f0c30dc6f8db703174863a13e3a636cb5edbeb42a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"3edae431ff77e83dcdf6bd1c2eb663743a93ce3c9b7cb3a54be975f2dcfb6d89","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"49de7b46b9b3907a2fe13d87c9a7bca344c3c0adfafa837d7453de185aaf1ace","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"5108d97a5606eaa075b1970aaae97cec51687933e02c22261a058ccdac27ef62","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"54f1d9a0c6cd9f8dfa1399daa2e94fc81a31129243e5518016205a837d888150","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"588cf8f8147dbe85dbbb259949b40126c92612806398b5b6d536d46ddea65806","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"69e9e9783ce65a18d776db022e2404a222c01c581364d8a2e36b96160faf315d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"740e4b12b18f7ec3abcb59e913c3aac96c3fe7b74097b2e379548b0dd52e587c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"8163767264e2f2a2fe4559b3d9e8fb700c29c3b53c21cdf589c6ebe8ef79d159","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"83828f42566f08f313e34932a1b6519381dc8ce91bdd407284e89ca8a43f066d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"859b8e42dc2c7d0eabfe6e9a3b77d5e0f6cb1580537ab75095d267e9761de894","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"8eef0192afaf496aef37e99995ae4991f9995de462839a3bc60443aa47866b4d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"8fb25f4e8459de9d88c4ce1a26e2e1d2d5c47224bd27ba215a6b00e68a8ec72d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"90830f3da3dc3037df5e95104b75732c5207b907730dc8220cf13cb0361fb3e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"91d27459306acfba0e1f7f9f09b3b58287ec53721ce8d78be8e6b545569e399f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"951b56bf930e1710f9c89251b9a92d9c434764ebbfa79c2d4c4a43729ba1ba7d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"9d43b79479d993e5a773883b2ba13571e76101d2f046d61293a9626dcea088bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"a2b30b04fab1e84ce5ed32dc490c3554a10695067ca8a8072c99db38f1901b87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"a34a7fa3e9d1f4ab0c95c2da0b2b4e7dffcf8ceca71238aa69ba9b23119b775e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"a46fb253a87328078c6120731291d64f564618c7a18531510d4d69f7772b50be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"abfa6c3bcad8ba9636e639bcf67ad9f2d23c2c5e09ef5ab42804ca777090585f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"b2ecf4c850ed1058543e7448874c8b4a516b8a4f9ecdf3e61ff1902b09377abb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"b551ad34ef00d3ca6022762cd167514b6045e84363efafdb07cb7bcc20894f44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"b5ba034a7e7faa6513a8362334aa2349a38e1f15cea70f857cb71a81269e4916","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"bf3ea225188ed6540e28e48f145437b61e1711f884c853ee13d450683d09163b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"bf612fbca19d7ba02181eea2697013613e523a9feaeec28f6e977ea886ebd7cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"c0f4bd59f8cf7d2eae33429f648303444db68d7251f4a49c971f628f3abc4da4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"ce692d08341314eead0cba8fe7a926c16f7053bb667d0b3113ec18c73d2c7ff5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"d2f9b1d6a4dc2c4e11c24b7d6d1e51dfcc55c35723b8be324aa376ec4482623c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"d2fb9b24ef565c017137a2c2415b0bb0571aab892399bee2bfadbea4eefa7b5a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"e1b91c0003dfe8287f87363a6d3af22ea60586676659b4a066752822f9c4c009","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"e484adf5a96f5e3e6610df6db11e90ef732c512294414f2f062b6ccb4ef4533d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"ea0bda361d89f660d4f4bcb7ff365c3eb3feab5621f6fe1e38c76f3ef2b369f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"ed979e02093bc0cb2a765eb52f1a44399359430e73772b4f7af673b82bcac847","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"f35c0eb3414354887d98cdba9e0ece3c1172c182dfac94f2753434d6783656d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"f71cf1119b72fb226f585c480f5ba078edf2dbda547dcc690f2857e234bc3df7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"f828621d2f402f351208ccd28b76304213210773de0719bb3782f81d89c3d726","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"fbf207641738ef9568ce0804e6984e482f34e3776ee8de4a2b6d63967e228570","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"ff171e14a300a05a23f6027ecc4877b5b8e3b0abb6ddefac1a264c34b7db81db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"ff472dab77219bb199206975ffe140908aae5d47dd2d0207429aabb27c7fab40","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","signature_id":"ff61025809bf6ec91ec3f088fbb166e1c4ec275b6978c0cc6520845e651c40ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"08cd2c9d0d0a279d9577666fa30616eb2fffced43794a9a0cbe6af2121ee841e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"093cc2aa5fea1f346a96d5f0ea9831c6ee5809e669abdd4b7bd35b19ecd160b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"0a0c1320a7eeb52e22629b0133ad70e66eb25fbd6465d07f9cbf35cc74fcc0f4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"0f18660f5a46ac4c593f38ca80319469d452222a1e40736b425b96407cb04bbe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"0ffd59ad5a9206eee67b4993c0256114d05784b143204f3158abb637f4b464f4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"19026dd0f0c7d922ef734965815bbffec0bf51fa0e45414a1f655aba9521df21","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"23b7b5a50a3a4247f6eb4a919de3dc31a515d185ef369550337728c80531d1e8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"2ab660ddfa8369b4721ec751c5848ba88b01437622b38fffb58b9c7e00f202a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"2f870cecfbe4a3b93f028e24986ae19473b1a868745eff93bbf9ea94a97f5963","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"30d9d60e7c74760d7faa5a8fb707247c19116124c5bd41f521b15348146b21b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"30f8c6e9234e350b78cfbb5eba5b86323b74f1bb878ffd37410fd172a95271b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"3a3b2089844386b0a0a0aff38076c2d9ff41f09239ab6e212e2eecb67e11e4fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"400c9669d5325333ee5b00dd3ee746c7a3e8995e174863705d405eddee9960f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"40333c4e13ca03923baac8958793fceb5b154ae21211840c3e1c24b09bc6f8b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"4fd29603fd19be00a0ae1b61c2047fdb3b264a2d611ec2a9c9c741ad8a510f58","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"53c29867a151f2a3aa1c86f74d859738b75fd581dc5e24467e5005adf199ae31","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"5779693e443f4472ec66a4e18cc539cbb1ef8e54ef73913a9d0f625d2b25444f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"5e7531a6aa4b5c85f4a15fe87edb3196b68e0a4d7da1f925dcfc53db2ae83778","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"610c5c09f47ef2b79cf62b2e9e05ee4c01040e492318fa5fa8db9f86cba76627","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"663afc0e88c526f3884b330979bd47c295e82e1413f6d52d4631a725ccc36064","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"75fd4b03adf9dcfe4c172c81d59735ae493913c35d28ff5f1912083a84e2095c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"773dbfb6f685d5e5b63a7dedb00c8a4c91e0dc3516e4bb3124784f9bfc141f26","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"77c7c1f85e68372813af9e1361f471438b4698b8c4434bb1b6430977bafcc9bc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"79fa9df8d01a0c8e6b37d420f1dae05e9eda340e5757ffeaa9b24c1063b7d0be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"7dcfd7bf44ca102ae754ee85eb930fa77e9750638d2650f2ce8fad900a35d033","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"7fadd67f7e0328d4bb88e243f1b95b75d78bfc521e8019e6ea3c95a2d031da14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"87b1fc3bd9a21833eb4626bdbe8c7b63fa5683e3e4f94de8e461cb50b8441b01","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"8b9bf78b58d24f5969338f5e907b5a2bf0d92a636d238ded487b936afe986bfc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"8bb74982db37588403d8fab7f5c99434cfdfd0abbd185c6a59b26d0aa99112f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"92d32a9837746d51292300852476757e0dc35eb5305c0ed369c7564340a6e5a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"94d96fffa7d1bad54fbf49591d13e60f8136da9a14459cf0cb84c99520a6c608","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"963b235af8fdf1bedd5e3b972e6ab535807bef24abc1e52a80df038e2705cd4a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"9aeeee06d3aead3de3e6ecdd1b73e69c83c7a37341b72ba12b0806d2719fa0b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"9b29a25bafc368764b1fbf73ecdc9c42531c8a9fe67214c77aa3280f5bd8b347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"9c1f1351192de758f7bca1386392fde30abb6ad8f1a10d99cb0815674c05ddad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"9c3293a96ffb6da050c034467869890467ccf8873332a3ed745d4801459b8224","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"9dfaf7121853481919e3881b0537a8956337c97f19dc90ab9d3dbe992b8699c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"ae40a611550633022816bfd1676dd196df0871b8abf8197571dce589941cdd4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"b2dd435af22e606acd0a47900d299f3faf20c03eec978e782f8187cd48938371","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"b79324929a9fd8a945f88bb03ac5549cedb9784b551b7a418d0ac1109fec9be6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"bf88654640f8a97216d71818181a0e0345da2e9b0da5b3d3b128a3269a0d46dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"d2dd8eef88efaefe9170ae4768c47c553cb499e079e1175e82effb4f0928e81f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"d5fc51d6545473d55401f281e1b238b52f1df0052b2aaf586405f45c49dbc916","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"d89a78d5abc1d60c48b13963b72095607403761742e35135f365f81a76a1ce7f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"dc2f8cd48d5662063b0fb3fb07c3a632eef82666036b5280cfdc2d6ea40562c4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"de0d88e725201398a26638900c4b8f13029239b8fbd2103a4e1cc978f04ae9ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"de8861fd628cfebc6a63f45ed2dfe26acde63f3bbdef55af54ab3d2cdf735cb2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"e83901819481d1fab21f58a929005995f6cb26767d7e4bb46151c4284dea9771","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"ee161ac567a0ca3fb7baae46e3708d3acacb178180936680aa4090a78efd9296","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"f88f9770171f82a1296dc291b410392d4a28d964776825a38e36f62a9e392a0a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"fa20f004eca313d06b7bf8f02b23d9c71c888bbea6946886f58e8920b5982412","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101446375962054656","occurred_at":"2026-09-19T23:00:33Z","signature_id":"fbd065e17b8759886ee623bc7ae73da6d1e4fa1aa9016ec8bfac65a2a02c0b9f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"021993ded3c59e43c3f839d0cdbbdf1e4b157760d4c9ff0d9f90a817c7ae4775","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"06d495a6a9fae6d1bc1c488918dbf4ddfe7c5ff233c7332e1429c73f548be3f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"093add3ee64d4954e3b94c3c2cfb4282cd41ee39c2822bd0e07eff879972aff8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"101cd4aa56258b4f8a9fe8936ce92d19ff857ae1db1aecb5c2ee1b5aa51ee62d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"13e33d3026a693b1f03e594d4940208a0f921a912296135f08978b450fb97eff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"159c0d62560f6fba5cb1754593d63544afe516a943d9c0a9e86ecb2ae121ee58","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"1820cf9d86a5ffa61ce244190c474f98e1ecb0da6974435f6cc1a986e5347ddb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"1a30934d0d58f623817dbb86941c4055e79a7419c2b2eaf4b127ce442274a4b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"24b79aab6523aa8acb8f22c871bf5564bba0683c9f3cbf359b5785e4bdc882a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"2a64a8dec609a2561efc7ab00a143dc1b632d8f84ccb46f78abf0eb8da7054cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"2b031c484e384d14c3ca6a8ca7601b469776dbbc3b0dd1cf628d08fbfae025bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"2d96cd53bb7ae31b0c5e1b367d0d0b22b6b570c76d7aaf899384344a97084f3d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"326c6de4511830e5cf10213fac3dd3a7b09a1786a9acb95fd3335cf494060113","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"3795f47d573e11b8f345b9b2aeaa39b561f9b6692f7b901198ae5c58ac068f97","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"37f2422b8be0d1a67ba134be7d4e0706bf04738f69f71816db068388177753e8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"43c31e5738e5bf02961fc364c58bde49316facd06ed0d720e4c24cd14ba64c9a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"46afb5d074aa209a6f1abd6824b1cd9c1486040fc99cf0d6d9d821e06b7a9e83","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"4aec705bb9e2a10433f462e1da12c54721e4b4dcdf4a512799d14c093c590361","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"4b3422b101eb06c0d9a9652ab521f096c8832c0cf9f0bd289c29c0b26e11442d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"5552bce84f58263b197dfec12a4c7a762bd959fd6512dd7bb44f85c193a93836","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"62d59fd5276ce2d85adbeb9c084016ddf445df1dd0edd67fdc25ca20a0b08a26","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"64d941203dc3624ac924954b3f233c151a0985c10c973fd7fe9378dd90669b6b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"661a2fa3b32aac188cd442775d3c8cd616d2f42c4c9caaa829399c6c5672cab7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"7137c8bfe05e065e60f258e57916a595216f790159d95b634d2dd2c88a9106c3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"71f557b339c85bed6f54bd28ea26c99862220c2fb39a0785fdf4cc27bda61f28","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"7443de46aaf45fbe19c1e7f5176f4af90fa13ec6c18ba35c9e97804d0f293b01","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"765f054de9daedb891daf25aaa0e02bf3137e93026fc221afac9c5975aaf2906","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"788d5921136bafe517b62e6791ddf2f3f2e9749cf9a2700c73d9f2c7aefc09cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"7ac42ea250e3e64994c3a5acfca542cb67260d81d725f3c0db1f7311cda6e9b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"7d5c1a2abf705bbb4ce88e6b9478f372a7216366052bdaf5053d8b991ae1547b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"7dfc6e8011ab6364138363d329b9c7973adebe42eb8f62ba981982eb8a0b6ea5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"81a14b59fd8899f07fe2d5df7671c4e8633459d97e478c75054ce865d4befe2f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"8338e56e55a189259058165803f3b4e2985e37d72458e33d47beeb2f1d6d7f42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"8a0f0cd0447701422b4533baecf66e1d8b4080ffa326157ef44c8f58bd7ccb7c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"8f2f15471bb2119e8f4fed62d0604b5e30dc3d83bbbdadd6daf88d604e06d461","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"926186fe32dd331064ab9f66b0896abc3216ea3ac6e64bae20a5e8e4e8cf532b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"9b80c2cbf4f794d4b388a0e167dd977229f910f1d62b536bfe8c52a42043e118","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"9bc278e4f312b88bace6fed4119032488299a51c813b0a8cb3b64a3023c00214","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"a10c9db482a3062db5b154f8e1fae726f29cde9300c738b2f14e70d5ff48cb9d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"a18977f6c3d71740580908ac0957a60b2798f0f9ca41808563106d1cdf816272","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"a35cf4597094a25da14531464c46a48fafb1031cde9c0e789088a4aa4833b847","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"a6a83890437c65a5482b260ba1a38cc91f1fd3c4258ded9f91aa4f08d476a502","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"a86135d811e5d67f5b5aa577a36e2901fbd5f9362014074fcfc98f11e5d24d4d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"ab90d23ef59fd46c72604b213822e81fb647862ae4690489b9ba2bb8e763a350","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"b139acf23fef5659a5f7adf0633aa9315b988ed27217d3acbc4d58d4b7e170ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"b1720118aa1eae0e384cbc89ec2ac60b36942cbcd8601fa9929cd0244eb999a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"b902724f0dee320b436bab1aa27050019a56fbc30e4f757df17278dd6fd9987b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"bdd291293e61883fadbd2a8309169cb15e9a2fa41e9df2df1d3fe35718051743","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"bf6d95f737fe627050799d89f96c289bff8066b76f3d1d772cae346925407c7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"c74079271692097374563b830a93ee2eb829febc7cfa166e479219f22ed8cafb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"d706b49740814b05b619a7490468c30aa1d97b77db44e28d38e2ec7d68848057","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"da4585f33d80028188a0a14971e89d65ec5fd2010337fd65c3553270bf268403","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"e01b71a158be7515a893103b1e31c88ce08c4ad1c0accc0c7d09d1ddd9a9a228","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"e6f7e807ed551618a513bb46d64d61b854c70331e3eae2cb8aac1e5f44d256c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"f522f1247ce440cb5d9318eb406bcae341eef0b24a3dc90ec7c4635e0bb17585","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101129282943193088","occurred_at":"2026-09-19T02:00:32Z","signature_id":"fe1b37140b7fcfaf273bebdbc335fa94fbb4a39a19f52989644221d821ecc1ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"085c6afa73df8e5dcbb8a9e0c59a48a53cb276afb92a752caf002a49c403af63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"0e7c33b837c87670cd35a11641aefa3118943334f501baf884bc7c01141848c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"0f82bd0c9700cf4a07c13d1f48415f495ffac7b49c2a7ae3841ea9c8faf79ca5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"10f37d08fe2d36b0048e252aedb2d548e4ca8db071149c417d5ce17b13fa36d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"1153df3f24dc8bcdeecfdd99c8141522fe1f0d9dd14f566bee678a30941ac49f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"1dd5e0dc7e15d356bc9a171360fd68f57e762e2e53e6cea924436fb68b6fb102","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"252c4afaeebb14c4115f0be666abd104e7adc80f40991bcc6beaa3ff85c91e84","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"25dbf965d6f20a1a7cec9776ec1e87245a3e50198ba091a7741329a88e0da378","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"265ce4c4fa80432d1672154f22571f629b7a60e7c351781f21d111b2d1880927","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"2740332bfcda380c3c02e8ee1f792b3568b04678a77023d1486f19de701aeb61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"27e6130cf36baa158d6711ebe28c73dffa820785f17bbc686ff8fc7d38fbdb8b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"340421cbdb7899927b7ac5fc7d0aa57d005e6eb5d784359ad31a36aed34461d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"3f18ddd10272127c408a7d9bde4eddaaa27e6f085f80cf7d94ab9be21a380754","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"42f7a4e6cab05c380f4d5d815367e49f29a7e4b22dcb36d855a4a05388b7e8cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4455b438d122ea70538db65aa4dcf60523e82a18567f93d3872e7891ddeb2d9f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"44de21e589e428af479c9ccfb6a72f778841ee48c0b271fb47903606d3baee24","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"483e21afcf9632ea6968f51c68da44c7e6f45d9a5418bb6f513b2bec6b1bbce2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4ba5ad9b16c83f433ab867cff2637f8bd36aa451460f9d15a478f5eb1eda9a65","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4c2e2bc46dbcb65a27b428b8f5c2170eb0f11ba205842dd52e99ae5bde4b31fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4d36c1670f69a8378c623f46ed5e24f3787ec0a5de39295ced2ebfe81f29cefa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4f0f437ca045765d2d2b24079491f21e1fd5a85db0b44958c8034b2e66199908","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"562b8f83f84dba1886b3065181ffee89aab2e6673b7333d41f76aef2eaad1f4e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"58f07f406d642a91c9bddbe8fdd7ede440f08a2dd03a6f5a5c819b498466cf11","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"5e002cf10865e514b0815eed29286f8da9237e5aa8330589afaeef395b289efb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"677da6ac04d265e57cafd47671de87d4257ef7d06221d84329ebe1ab461fca66","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"6adb067c216551bfe29f1e3b77d6113e9d3d513c52d7c6b6659f174eed5dbd71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"71d45e9348cce618a98ba3a9cedd12637edd7a5d0db05784f92b3cf08108bd98","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"71e62b511a0fe354af7045a804df32da837a8d8a1285d20120469dc2b9b474a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"8b5bf6763af34e2d91fc69a112089a080b1da4da152d93af2ebb7174e191d48d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"97623f5d3f0d0fd367349a611b76873782d9adb09766bc94fe24252159b1769d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"998b87f452d353eeb973a256989175c16ceae9088c3fe30ce72eb5beea463849","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"a4bd95380032fc5885c8b2fd10eb51f3df30c29355f6b414b5b33fef1056e9a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"a9d975f128c27c084e7e5fd23daa85c45a8d628e9193f529805c3f218b289677","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"b44a4a5abf9df504b29cd303adc2b536d491938b5012cb47d37469bc577abd3d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"b89878daa0c4d5d3cb3d93a46690c3d9ddf063236bf34cd2f69bcdf3bb5f245a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"bc25c842a1235045614d605acc90fea01e39dbaf7d68ff406505237e93f1e16d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"c0258f3c8ab984584c9b6314be2ccb30e5ffa2b206f4ab774c595c07d6b813f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"c3410d264cf0132fe4f17940cdf2ec821ec784b23b0faff95c971f7658965387","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"c68022f2f40c275eb8c23a59749c7f7e4af0d44ada9136f9192d94b24e86254b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"cb988d58c89113961f03df82043150415e9b6ed049bf1f878c0c0b5c9eb97486","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"ce5b35b01c844ca412a4cc92791058f44a790b6ff65cc19fb352ae11b15d2ae3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"cf3269165eebc8b281476a0026c2b3449c115912f9f0f7f746f7524bfed7b58f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"d66e9a51e6356e3da95cc4251424afb6ce526cd142562c82ca1f05f8b3157c1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"db766735f9e579f4552ed8a3c29b312be7add829d329efde1debee620ae19b46","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"e1a746b36f9a4f74e169ab5b044f8ffbb4e94a6ac47cce6553d8ffee67ae68fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"e5ac7068f8a4f91ce6b642a6a37bd31dd8ace822a1805ae8325512e343c782e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"e94453651233184d87c4057afb3b9acb271e36b21383edb286efa6f234f89bef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"f0973137271b8ecc421edadd409b7fe63accbff0882cef646035b79a0aa8c22e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"f0cc45ea9fca99b68f2c8c623264abfb0139dd3f1bc78462db9705e9cff98ef3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"f6f1275a16afdec341158ade8fbc4ce79c93972342a64ac61c74f29d86d94b33","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"f7953744450082a3ec5bc94fd5cc7b0b148b3213f668f2afb80a43e6f0953ba9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"ffc3ee48d637c5b8ce881fb48a931d89a1627b9e6bc94e2411754ee778248a76","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":5,"runs_affected":3,"failed_runs":3,"run_impact_percent":15,"anchor_occurrences":5,"after_last_push_count":5,"also_in":["DEV","INT","PROD"],"trend_counts":[0,0,0,1,3,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-h29zrk\" for cluster \"cluster-candidate-4-21-h29zrk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009df3c0\u003e: \n    failed to create NodePool np-h29zrk: failed waiting for nodepool=\"np-h29zrk\" for cluster \"cluster-candidate-4-21-h29zrk\" in resourcegroup=\"rg-candidate-4-21-k7gt4s25kh5h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eba02645-cd7e-4330-9e90-bb7a800d75d5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eba02645-cd7e-4330-9e90-bb7a800d75d5\",\n      \"name\": \"eba02645-cd7e-4330-9e90-bb7a800d75d5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T02:38:03.543368376Z\",\n      \"endTime\": \"2026-09-18T02:57:09.03735448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-h29zrk\" for cluster \"cluster-candidate-4-21-h29zrk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009df3c0\u003e: \n    failed to create NodePool np-h29zrk: failed waiting for nodepool=\"np-h29zrk\" for cluster \"cluster-candidate-4-21-h29zrk\" in resourcegroup=\"rg-candidate-4-21-k7gt4s25kh5h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eba02645-cd7e-4330-9e90-bb7a800d75d5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eba02645-cd7e-4330-9e90-bb7a800d75d5\",\n      \"name\": \"eba02645-cd7e-4330-9e90-bb7a800d75d5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T02:38:03.543368376Z\",\n      \"endTime\": \"2026-09-18T02:57:09.03735448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fc2640\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4xfvdn4xsffw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98b08bb9-0eee-4840-9535-91464d8458a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98b08bb9-0eee-4840-9535-91464d8458a2\",\n      \"name\": \"98b08bb9-0eee-4840-9535-91464d8458a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T05:40:34.02322608Z\",\n      \"endTime\": \"2026-09-17T05:59:42.272250066Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fc2640\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4xfvdn4xsffw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98b08bb9-0eee-4840-9535-91464d8458a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98b08bb9-0eee-4840-9535-91464d8458a2\",\n      \"name\": \"98b08bb9-0eee-4840-9535-91464d8458a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T05:40:34.02322608Z\",\n      \"endTime\": \"2026-09-17T05:59:42.272250066Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0035265a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-rcr2wwpjvcjv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b397553-ee61-4674-b26e-e3cf8ae2dc9d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b397553-ee61-4674-b26e-e3cf8ae2dc9d\",\n      \"name\": \"5b397553-ee61-4674-b26e-e3cf8ae2dc9d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T05:39:36.012772677Z\",\n      \"endTime\": \"2026-09-17T05:58:42.168749593Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0035265a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-rcr2wwpjvcjv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b397553-ee61-4674-b26e-e3cf8ae2dc9d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b397553-ee61-4674-b26e-e3cf8ae2dc9d\",\n      \"name\": \"5b397553-ee61-4674-b26e-e3cf8ae2dc9d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T05:39:36.012772677Z\",\n      \"endTime\": \"2026-09-17T05:58:42.168749593Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2100766832767012864","occurred_at":"2026-09-18T02:00:17Z","signature_id":"16ebc00197f5d484a478287dbcf774e95611e6748652e8b6528ea5b7d11de1a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100452501269516288","occurred_at":"2026-09-17T05:11:15Z","signature_id":"573bf63f25f718e6e10d14bc2ae7830ca909cb9577c72529929dab5db0a4d7ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100452501269516288","occurred_at":"2026-09-17T05:11:15Z","signature_id":"cc190a2c1395e4f5ad1b156f9318c22909f9bd03dbf07b404cb1ad51d8dfd21e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100452501269516288","occurred_at":"2026-09-17T05:11:15Z","signature_id":"d97b1d417fbed487ffa515ee9e94f38afd2685c1f5cc55df9fad23cc5afc6bc9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"18ab22781017136dd055b840d502183c7edf1caa41f67dcadf9fd0a7857c8081","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":5,"after_last_push_count":5,"also_in":["PROD"],"trend_counts":[0,0,0,5,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001d4bc80\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-56f2xp2w67w8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001d4bc80\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-56f2xp2w67w8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012fd950\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-57279mxs72dc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012fd950\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-57279mxs72dc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f2fd40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-g8sp2mm6ks67/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f2fd40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-g8sp2mm6ks67/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"3799930be8939ca48534a28b262ff33328138426e7cec1b9fb2793e6f4a95899","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"474ef928e4bfaa163ff38e770c6435689354cdc8821b9a05ec4272d8ad60c351","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"7a42cc1723b1b8bf9843f1aa12dd8baec4448f6917d7f084b3a4b154baa63afa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"8be57f60e7cd91c124c72d6d01098c6e1b8b853355d51b79f6b1bd55c240efdb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"bd4c07c42b25870c0f1baaecfa45c3c668afd2f7347cf758e572515c16e421fd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":4,"after_last_push_count":4,"also_in":["PROD"],"trend_counts":[0,0,0,4,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004d4640\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-c8fd7f\" in resourcegroup=\"rg-np-version-upgrade-c8fd7f-kdnr5znbw57r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-c8fd7f-kdnr5znbw57r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-c8fd7f/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004d4640\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-c8fd7f\" in resourcegroup=\"rg-np-version-upgrade-c8fd7f-kdnr5znbw57r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-c8fd7f-kdnr5znbw57r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-c8fd7f/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002c1a920\u003e: \n    failed to create NodePool np-pbnkzl: failed starting nodepool creation \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\" in resourcegroup=\"rg-nightly-4-20-6c9ns9nbgrzm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-6c9ns9nbgrzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-pbnkzl/nodePools/np-pbnkzl\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002c1a920\u003e: \n    failed to create NodePool np-pbnkzl: failed starting nodepool creation \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\" in resourcegroup=\"rg-nightly-4-20-6c9ns9nbgrzm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-6c9ns9nbgrzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-pbnkzl/nodePools/np-pbnkzl\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000facdc0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-m58t8j\" resourcegroup=\"rg-np-dg-minor-m58t8j-trxh29mwfdlb\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-m58t8j-trxh29mwfdlb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-m58t8j/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000facdc0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-m58t8j\" resourcegroup=\"rg-np-dg-minor-m58t8j-trxh29mwfdlb\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-m58t8j-trxh29mwfdlb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-m58t8j/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"3d8d555b219d0ed8c9661c956fba32429ade34240dfb89932e7b6f838ec71e56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"51c0a7fe19269071bf4a86bc72b032196ba0e6e4e142e092ab1f3aca5f282e06","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"a3bcc0cf85a56ec12c072c420e0931c744e3a86ca315e7af1b9dd0b787fa5b91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"c259628d57d4675c02d0130322edaa375c878722b676fb3290dbafba51543aa4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"5c1e65a74237eb97d61c283de75bcde39935ce10b1226f1bc067c35315e03a3d","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b5a80\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-2wnkdzqz6d9w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2wnkdzqz6d9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b5a80\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-2wnkdzqz6d9w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2wnkdzqz6d9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"5c2b3fa28b82ccf14825b20d7135a7cb3d0c7f8bdbad597c1b9eba4c000e0333","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"ab5d80437500581ad7ce3942572aca8a7c5ea9eda26f4367215f73be1501d7a0","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"npupgrade-4-21\" in cluster=\"np-version-upgrade-cluster-nl8qnf\" resourcegroup=\"rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-21 in cluster np-version-upgrade-cluster-nl8qnf in resource group rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.33\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002292870\u003e: \n    failed waiting for nodepool=\"npupgrade-4-21\" in cluster=\"np-version-upgrade-cluster-nl8qnf\" resourcegroup=\"rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-21 in cluster np-version-upgrade-cluster-nl8qnf in resource group rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.33\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002292870\u003e: \n    failed waiting for nodepool=\"npupgrade-4-21\" in cluster=\"np-version-upgrade-cluster-nl8qnf\" resourcegroup=\"rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-21 in cluster np-version-upgrade-cluster-nl8qnf in resource group rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2100766832767012864","occurred_at":"2026-09-18T02:00:17Z","signature_id":"ea0f58ce05963824cd5712a4a308420855630745ad3e354951e106ba980a6661","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"9f24424c672066cba3c7cb45f333ebe45b01b356fab007f5a65bee0f062d44cf","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["INT"],"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:148]: failed to verify cluster \"cp-ystream-upgrade-4-22-j5vkzr\" is viable before upgrade\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015d2858\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:148]: failed to verify cluster \"cp-ystream-upgrade-4-22-j5vkzr\" is viable before upgrade\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015d2858\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100534031866138624","occurred_at":"2026-09-17T10:35:13Z","signature_id":"aac82bf00f815d526a50ddb15d7fc450980f914b55421ea2243e84387908cb3a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-13..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0016e5a88\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0016e5a88\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"77617baa5bf45330c8ac3ac9717b66793245c58ebcc17c8a3e357cf43d6f0bf5","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}