{"meta":{"start_date":"2026-09-12","end_date":"2026-09-18","start_at":"2026-09-12T00:00:00Z","end_at":"2026-09-19T00:00:00Z","timezone":"UTC","generated_at":"2026-10-14T12:04:06Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":730,"failed_runs":607,"raw_occurrences":1099,"matched_occurrences":724,"runs_affected":232},"rows":[{"environment":"dev","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":187,"runs_affected":76,"failed_runs":76,"run_impact_percent":10.41095890410959,"anchor_occurrences":187,"after_last_push_count":59,"trend_counts":[0,0,0,0,0,71,97,19],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":76,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":12},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":8},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":13},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":11},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":9},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":9},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":9},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":9},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":16},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":11},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":12},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":8},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":13},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":14},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ce8120\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-4k8qcm9nwvt9\" to finish creating: GET https://rp.j9556480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ad50471-887d-48c6-8d40-87da15772f0f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ad50471-887d-48c6-8d40-87da15772f0f\",\n      \"name\": \"8ad50471-887d-48c6-8d40-87da15772f0f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:24:52.179447145Z\",\n      \"endTime\": \"2026-09-18T17:53:56.374800037Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m13s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ce8120\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-4k8qcm9nwvt9\" to finish creating: GET https://rp.j9556480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ad50471-887d-48c6-8d40-87da15772f0f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ad50471-887d-48c6-8d40-87da15772f0f\",\n      \"name\": \"8ad50471-887d-48c6-8d40-87da15772f0f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:24:52.179447145Z\",\n      \"endTime\": \"2026-09-18T17:53:56.374800037Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m13s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fae660\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-77t5pbg7l7p6\" to finish creating: GET https://rp.j2056320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9382644d-58f7-4d28-9aa1-a5cbc03fae7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9382644d-58f7-4d28-9aa1-a5cbc03fae7d\",\n      \"name\": \"9382644d-58f7-4d28-9aa1-a5cbc03fae7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T12:53:37.696598816Z\",\n      \"endTime\": \"2026-09-18T13:22:40.932765907Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m26s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fae660\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-77t5pbg7l7p6\" to finish creating: GET https://rp.j2056320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9382644d-58f7-4d28-9aa1-a5cbc03fae7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9382644d-58f7-4d28-9aa1-a5cbc03fae7d\",\n      \"name\": \"9382644d-58f7-4d28-9aa1-a5cbc03fae7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T12:53:37.696598816Z\",\n      \"endTime\": \"2026-09-18T13:22:40.932765907Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m26s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015787a0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7x7svb59cvl9\" to finish creating: GET https://rp.j2056320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/79d978b9-66a2-443c-bcf8-f3536818238d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/79d978b9-66a2-443c-bcf8-f3536818238d\",\n      \"name\": \"79d978b9-66a2-443c-bcf8-f3536818238d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T12:55:05.057839777Z\",\n      \"endTime\": \"2026-09-18T13:24:10.88915368Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m15s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015787a0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7x7svb59cvl9\" to finish creating: GET https://rp.j2056320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/79d978b9-66a2-443c-bcf8-f3536818238d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/79d978b9-66a2-443c-bcf8-f3536818238d\",\n      \"name\": \"79d978b9-66a2-443c-bcf8-f3536818238d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T12:55:05.057839777Z\",\n      \"endTime\": \"2026-09-18T13:24:10.88915368Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m15s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","signature_id":"99f885a8a863b5de08d9591137eee29d3c46e0095bea1b9e8046d833c161bfec","pr_number":7010,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"524cdf887bee162758b9f6eca5510d2578bb6ecf654e9948629c9452eaf1782b","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"65b5dbb9967304eedefd97b212163b3f8af9b0b2bb0188919a3d12f3a6538fc2","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","signature_id":"78800d7f9bd7ce60d82097401bbc1a1b346c5eb770d7eeedc635c6d48abac870","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","signature_id":"ba73d1a54ec8e3a0b02523dbd4276ff942ead7e6f9b4e52703889f225a290272","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100867050745368576","occurred_at":"2026-09-18T08:38:28Z","signature_id":"2e53b9a5382ed0f0493cfc0c48e9bf7aa226835370880aa8b0e3c984f3d43f8d","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100867050745368576","occurred_at":"2026-09-18T08:38:28Z","signature_id":"bb9a6198465bbe86bad7f0728097a5e8139953db856c6c8aad613b2d535d245f","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","signature_id":"001338be47cf8dcaa3fc3c500d6a6dec5a631054e1ca9f9d544c49d235b2dcb7","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","signature_id":"782ec114fda729c308336015b9ba3ea7dba7299080a2ca46beda618ff211a392","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","signature_id":"abcb5fca9b88f5de84bce4c63659906c7070748eea55c112da30810936b0fc97","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","signature_id":"82775d4e1d46d1566b3f4eab5619ed61b625107edb899f3a6d4e9d4864e726a1","pr_number":6984,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","signature_id":"925fc6ad3116b3a4af4d42c985323051e3b386c99d2bbff7d1917ed9c6b7ef67","pr_number":6984,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","signature_id":"a2d0482738306402a9aee374779b77d199fad57d181dc39a001fc3fa8cf50f3a","pr_number":6984,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","signature_id":"b838f89ca9ff0822f2ef510e9178d1d320d4033a5d5cd3ca5c98537110704efb","pr_number":6972,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780897212043264","occurred_at":"2026-09-18T02:56:11Z","signature_id":"c3a814dc2b1fc08b7b4fdb53010472ca317b4e82bd862888bcc25a0f8b4091ea","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","signature_id":"2e205df2fc76c59d7d5bced445a9af5fdbd8ffc0a4321fbfafdc448b6041ea0f","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","signature_id":"993ed1b03e808393e377627c70564f00855cc6cb2d99bd1882ef2f7b7c73270e","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","signature_id":"a44cbebfe0aee283a67f51149529d8530cd9c5797c013288bce57d67f21ed3fb","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780819697111040","occurred_at":"2026-09-18T02:55:52Z","signature_id":"a093a69ee54b8dd613584cc9b286724efd055bbbaf92d5a169de74315234601e","pr_number":7000,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721105009381376","occurred_at":"2026-09-17T22:58:35Z","signature_id":"d601eba3abc4f82457501b40b90cc5d7561cc3383a0ad74d5298ef8e79a87795","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100714303135944704","occurred_at":"2026-09-17T22:31:33Z","signature_id":"7468e248b9c77a11eebc376f236b7610c67fe482e15792bf83562366e5934d5f","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100713215490002944","occurred_at":"2026-09-17T22:27:14Z","signature_id":"b9f0cd575efa9d87456c559803742a6d89fb91600ce45815cecb4689bcc702f8","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100713215490002944","occurred_at":"2026-09-17T22:27:14Z","signature_id":"db652501caa7a4389b880b5afa48aab924fa2baf6e02836e6489573d8b826dbb","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"6dff3d6d762cfe52b458282a2d61a5824494a4b7fbf9f3aab1ba2e26d05698c5","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"7bfd2cf527b8d88bab96af9ab4da562292d265e26e2314dcdcf6bffc6b100ac5","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"d74cdb3a0399ea27cc6c2ee2e3e86b457bc6ff248822afc15a1f630792589716","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"e593b0bc62b406e1f219043a4233fe6d4b2427734c6df50023d50f3222509f40","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"0a1d46d442635db3b7061c82189b8035af60a48310a53ecc646e31620910a1cb","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"50d86109b369a86d21ce7f771945b06926a4c38d2d49433ad0d362386d5132f7","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"78fecc2305e80f61dc9910998cc7c318760f78b41aed63eb7f46b3ce837e0d9f","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"aef26cedd47d167588772783661deaa26628c5b57bcb923a1a0a520ea749663a","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"b07610b43263bb7e953aa2613bf43c66ef66f67a6d400930f3acc7d9b2871dfc","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6998/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685757428535296","occurred_at":"2026-09-17T20:38:07Z","signature_id":"f8f97e2decd1a3d48fb001cc1c33150efca6eca10d42b34bfc0118d3f5dd19c8","pr_number":6998,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685654236073984","occurred_at":"2026-09-17T20:37:43Z","signature_id":"68698c4e2d81e96a811614a8eb209ad22dc4f07b9b07bee4f975ea46939d038d","pr_number":7000,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569791986143232","occurred_at":"2026-09-17T12:57:19Z","signature_id":"62a250c534c20b5ad145d249c89f89739cbae0c9b5704a8277cba737ff1f0047","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569791986143232","occurred_at":"2026-09-17T12:57:19Z","signature_id":"824964c0c575d45ef044ea907156d5bdd2f98d69978b43f2f1fb3b622414875f","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569791986143232","occurred_at":"2026-09-17T12:57:19Z","signature_id":"f24c215311a297177eaa94e5511fcc51ba57d864ab61313b4419552b1cf74930","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569759195074560","occurred_at":"2026-09-17T12:57:11Z","signature_id":"903f3f622deb52a1f1d8dea7e418d3ea7d8db5819b70eccb79ae8ea0a5299dd4","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100566804978995200","occurred_at":"2026-09-17T12:45:27Z","signature_id":"1fb4041dc59d81c7e183948830ae7ac1b2783b718bf3d7631b04090e55e3f847","pr_number":6886,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100566804978995200","occurred_at":"2026-09-17T12:45:27Z","signature_id":"3a922f4d19da47f001e6311e71c0f1a6d255911648780e0fdc7768301e19e9b8","pr_number":6886,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100566804978995200","occurred_at":"2026-09-17T12:45:27Z","signature_id":"e828cec50a4c240181974579c6679994f89f3d798f5316edd2d635df29854690","pr_number":6886,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100563400982532096","occurred_at":"2026-09-17T12:31:48Z","signature_id":"9771d1687cb82ab4b14c0f80173555cf109826201233928e90423e0171bc2724","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"85532b20bf57446a7d41586b4163ec6f19b5563d86b6207d2b7d5ab6951e2651","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"ab6e130ec463d100706975248c724bd66e5d7d3642c918a3434eeabaf4b5aa60","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"b8f768f8fc135387ca0e0f20dc96f1cad4999bade1e76fff9ca2066634cf47d3","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"cb6c78925476f970ec41ca2cc86883f3fe25eedebef129a57d35868eedaf968c","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"ff4a343919b36fa3670725111608ea3b35a12089dab63a806e3c5e121a1fbb9a","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"236cb0e5207b1956ffac77c17fe63e39c6df7e6a37c02c56004d161df8a008ca","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"39a1e24d4800633f0d0b42d4f18189154efcf362e824edcdb07cb8b8b0dd68d9","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"401c9dc75b4b5ec84bc7eacc4dd5a3649404e5b9bb0cad164dff2ac11981d6c4","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"4dab39d1b1d724fed25dc69d84d189cab2ac6a8152b77e555635e3621d133e55","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"bb7a8e5fa783928e378dcc846c79e38f12b390e8f34e30b5e6460dbc694c3af8","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"ebcbd7286e42fab84ff7ea3f9b915f4d4683f1309800106ed6220b22cd9d54e2","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6994/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100557383175507968","occurred_at":"2026-09-17T12:07:52Z","signature_id":"ee498f50c2b2ac982c301f4ac0cae3713723036b9fbee8e63fbb794dec859320","pr_number":6994,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554708664979456","occurred_at":"2026-09-17T11:57:12Z","signature_id":"d54234423202acb80a8cc7b79b53caff0b960878b2770d3f2b50c496f32d7589","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554708664979456","occurred_at":"2026-09-17T11:57:12Z","signature_id":"f4470bf38da6c976133b96af81d9ecd329780f2aa1f6bfa931cdb8bcafadf34f","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","signature_id":"8063e53d23cc71e09e9af725e9b15fa0500e6877a011fce15107ebe822270942","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","signature_id":"8cd2901e21933a26dee207d65687d26f76901c588c4dc077ff1cf8179eac46e9","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","signature_id":"c31f9f905858e3bef622b1331144e2d854b98bc4c9aa6c6098d97c66fd0a17ca","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554684329627648","occurred_at":"2026-09-17T11:56:52Z","signature_id":"9577b3508cc1b362f70a7592604f657ac43e72478f7b4c69c3b22190b77112b8","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"2cb0fa49d019c2b0a11abf2d3312cdc656ba3974fcf4c96aa2f18765ffe8dee0","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"61b54a22bdc6f8db1004cb461b9c1ea0ce8e7bb226818ce98016843218649c01","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"6989180271f1d05831c6e509c6cc427effaad2b85f3bbba286006443200fcb35","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"78cd64b525be75a4d9388fde337bd307c022a2d8ce21d0ffc33bf9360fed27a3","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"7cf6bbc9da68baad7e4462e2758e70a60f2061703be4a96f8198d3c3759e5683","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"969e77e201e0ea5c3383048290794e2481a2bc4e0d76aac6da2b54ff4ff6694c","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"a21f245f7a5f77e335623baa9b0ddbede34c06b521c1c7c4d490e127ce4037e7","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"cd184104708d45339d6bffb3c0990b1efb5eafdada1ae446f6ffcf492fc8f488","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"f0d309a90401bc7c464e4f86124fff842aa1f093303fd8bfa28b66ac92405fe2","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"fe2081edacd08521e3b4731165e95ed1b4f17a242e9810e3d19e5b07192590fb","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100537768756121600","occurred_at":"2026-09-17T10:50:04Z","signature_id":"0d5854936e81c77013558f96a12c534160fb9ce058180d02b89c4d9a3544856d","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100525488945500160","occurred_at":"2026-09-17T10:01:10Z","signature_id":"3596b17636b3dd9dc619bcf3bc7003efbdc9033c334b506d05c614ad30a25383","pr_number":6972,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100525488945500160","occurred_at":"2026-09-17T10:01:10Z","signature_id":"b8057328bdd22a04aeb32a86271770e8c39d4d2cd6eac75a438bd766471d440b","pr_number":6972,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524951428665344","occurred_at":"2026-09-17T09:59:04Z","signature_id":"8bdd74c61a93beb235e7d38100063c0ca3d930717abd8601b9cb4dac72149e51","pr_number":5724,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524951428665344","occurred_at":"2026-09-17T09:59:04Z","signature_id":"d6f32f2cee9bee5817a4bb9c6f37367a9340014362f5ecc9c490646d7eab987e","pr_number":5724,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","signature_id":"7cee4e5352bcdeee9af619ecd98980da8f5a54689aadb5ebc237dc1aa40f7036","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","signature_id":"837607adb29247003f557d4cb6ffdfdb8aeac843b72cf365de5b9d8cdce185b2","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","signature_id":"b097d1bd6e31b2848274482e1d375642d3af07f55f2c3adb043f6e076a9933e1","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","signature_id":"b662a9003e08c3ef5e2286107d4a3536bcba752aaf78844948d28b110eee221d","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100520057816748032","occurred_at":"2026-09-17T09:39:42Z","signature_id":"2cc7ca1e205a8dfaf045d2763125c668a7279c4261f08b6ddee57e833c36210a","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"2cf84524e6ccdd0a761441daa39b744a2ba3952626e9461b7d45c15b3bcfd754","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"3f28cae00bcc4e17acb25bb050d65aad4ea199a510b36c178d2d712889664cab","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"acc3649bd8151c983690e39ec8f25e1c5473a348f8d0898d6d79bd8563a49a87","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"dc4d8b8d1deffa568a3e69aa51d30ab552205c7e9c9f140c68e72f873ccbd501","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506297609752576","occurred_at":"2026-09-17T08:45:01Z","signature_id":"2ac60a748539f98f2c88eefc3d7d9d565740acd60ee833fc9852acdd1cfcf35e","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506297609752576","occurred_at":"2026-09-17T08:45:01Z","signature_id":"6be75bbc95c50cbac96f78528dfa07add64f7a96b8fe35fe1510e1615554ba67","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506297609752576","occurred_at":"2026-09-17T08:45:01Z","signature_id":"7f2b1821d50e299f9faaaf1ec8623b52220c8b011f2741480e4d05563a5a26c3","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495749199433728","occurred_at":"2026-09-17T08:03:06Z","signature_id":"706ec129f8f00b0fced3839fa11ea395190e7f746c1a75084493342ffbc32a92","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"1254becb25675dbd9ede114df6e32fd10e76cd84f176a807e3d5dc72b936cfbc","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"1f2d8a520d63c686030e3b1f47535fed1c6501ab4abdd19d36fb0c76229a1cb8","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"32d4bd52ce4e4bf4bbb0c374a620144f8370e122a59cb1cce2ba39c24a22fcac","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"71eb266b069aadc565cf2dfa5c38d0b00bec544a93f9accc8e29672f14a64cd5","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"833379ae7b454d610a7efbf91b7df6a563effdab3e7cab0f6ff3de3a51cd9ae1","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"88399b6110d57a9572b0144a071be4bf2ca4aeadc0f450fbc20ec223beef26bc","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"ced0fb0de9a8798a8f89849b4e849e1ad6b42de9776ac8d0c57c01c8648db002","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"ffd6f4632b90a496bd463d4995c3c420f976f2007427581611b1c7bcf4dd4b9b","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494074514509824","occurred_at":"2026-09-17T07:56:27Z","signature_id":"109feb9a36dbfe6e2dc23c6d08e79142806f38590e9a903022f2d16ea982a69f","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494069930135552","occurred_at":"2026-09-17T07:56:26Z","signature_id":"38588499aa79b332fce0da4da61a0607d4dd3683eb3c9cf92c9d624bfee5f925","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488796448493568","occurred_at":"2026-09-17T07:35:28Z","signature_id":"d26e879298c7f737b9ae16dbe160a711548fa3f84ab3c2a0048bc582a13e4076","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100479104343085056","occurred_at":"2026-09-17T06:56:58Z","signature_id":"008a91fd3e3e5f5195f74645a3769624ed2be2ab43fcea742cd06f90df27599f","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100479104343085056","occurred_at":"2026-09-17T06:56:58Z","signature_id":"6b862675f286ba82e4aaaeabbdde3ab8b1accc783182f06adb944ac909b9f88b","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100438339525021696","occurred_at":"2026-09-17T04:14:58Z","signature_id":"82d6e28574c7f34a07c333ca94f997752b96649f6b80a188942914d02804a6b8","pr_number":6986,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100435900084260864","occurred_at":"2026-09-17T04:05:17Z","signature_id":"2938ba362ea03badfa4fef3a5a8b455d8db3137be286b0a32f0e06cb64bc56e1","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100435900084260864","occurred_at":"2026-09-17T04:05:17Z","signature_id":"89c35af4f14189b65c64576cd7490a0088f1c8ab485b614bf43cd6ed4c19e683","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100420606720741376","occurred_at":"2026-09-17T03:04:31Z","signature_id":"72e9dcaaa03f26fa26de2141ffb6133bfeb9f62e98e7bbfc149c580a12c625ab","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100404999925272576","occurred_at":"2026-09-17T02:02:30Z","signature_id":"17d592d037b2f2a804c6494b323dd9c8826c7ee66667ea66a41a04ee3ac208fe","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","signature_id":"1c3549dc55bc6c387adb5760f50c6e243c8a43f7b39ed0bf467dc5a0511968bd","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","signature_id":"3287e14a7b0a164b12f072d969ca8183f1647cf1f1f00a66de841c81d64ef6e4","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","signature_id":"a5eed4f538e16bf946e6c1d7bf29b73fc650cf9bfa21cbd95eec591f73113f97","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"0d64bfff34a7c1f61cc366edc7ee533840e5653ed9071035a92b925eab037b24","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"2913423376b3c9240235607ee0d4c675506def5b909b1312a4bdad97516cd24d","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"428648ff0c0135e57b228a05c39d11591ece01ed750e930d86d83fe847be740f","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"49af2dca8e5069846af20cde9f9397824ce26e8ea2d79c52e7c7b30e8faed262","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"721faaab8ec90430d24300487ededdb0a93721318cf4132ab4ace8fde5518506","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"a2ee109ec25c2b87338aedcebe4c8cc584f5a143bdc97ee69a387447d2c549b6","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100385816462233600","occurred_at":"2026-09-17T00:46:16Z","signature_id":"21b15360431929bdb593a2107c3896a1ca85c63523d6f4c20410fb74f623d0a9","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100354655547559936","occurred_at":"2026-09-16T22:42:23Z","signature_id":"f2b94e7451cbc051fbf0de3b84883a92be81c7f11c25cc3c65ce003806a07176","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351996258160640","occurred_at":"2026-09-16T22:31:53Z","signature_id":"6606b623ae18c88ac8a9b2ec7e9a18026cd7fd696af00a64c201797796b23884","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100346022097063936","occurred_at":"2026-09-16T22:08:08Z","signature_id":"40918e34615df9a134d5bd8ce9ad3cec41e9fc04a925911878e36407f747a316","pr_number":4878,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100346022097063936","occurred_at":"2026-09-16T22:08:08Z","signature_id":"be79ce9f9e2a3c3c1a4622c360ca3519703f1be9bdd39e307bbb6ae587836434","pr_number":4878,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100344321164185600","occurred_at":"2026-09-16T22:01:23Z","signature_id":"00790225a1ec5d2f2111abfe85ea313169365d1041de24fbe4fa745473e8e330","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"0f792fe0731eefcb63f38f15bca9a1aa450cefba466c1d19cff0ae8ed18fb263","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"2698bc1670e697e5f02bc62eaeda2db20989111b43c6c7fcc9af92e68d70c247","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"3728e9e0a9b25fc2b96787f8795446de81019e99ef3c6a67f320e9809b7bdae3","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"50dab7e83b7442f97fb4586fd719c0b5802925b2f02cc972849692995a093dd0","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"79d660f1bacf21e2718e2b4460bbd751ebcedbde1fed3611a2e8b06be51c9b26","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"8d9c0886049dce2e52d531fad148d6ed84eeb2446b9625587b295b408c739bc3","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"a52a7332954a9dce3940383eb5861ec4596c82a180f58ab40695aaa3094465e8","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"fc5496eac86ff2a1981e1d0213c04f2b8c1f98c1155641ca8fa469251820f45a","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"12b8d48d4627f15facb797beba81c6d3c5baa8cff824ed1576ee1a132af8dff4","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"27af66bf8605d41a0eed60544affb038a78040a481edac1756ed6ae13ca1bce6","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"6d4635ee801aeb366b63d020c6eeb3ed381519a10bc1a02c8e00a5d443d6550e","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"e798e0c9ad920ab5c7e8728fb7d75c823add0086ed40ac0f3ccf60ed14187ffe","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100299874200195072","occurred_at":"2026-09-16T19:04:46Z","signature_id":"5264dd1fe40eb5bea2fb2d3c2020b90fcd8c767922fd709e8151a6988e36b5d7","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100294017282478080","occurred_at":"2026-09-16T18:41:29Z","signature_id":"5b5a6e4f2745283c541cfa867e89616d0e7a33b663b83015979c872710cb256e","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","signature_id":"131d9678ec571fa272cfc7362879effebfa9b5e73d92d8d4799c9e8c4dbac244","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100262131709513728","occurred_at":"2026-09-16T16:34:47Z","signature_id":"4d2c343c16f83436cc3986db89b356fdfb3e28de9ebaa9ceda587e5572a6ffc6","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100262131709513728","occurred_at":"2026-09-16T16:34:47Z","signature_id":"7516a34fe822a219e86822cf9a1b8d51e620d585523f3da6508b9fb567e9b286","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252533153861632","occurred_at":"2026-09-16T15:56:39Z","signature_id":"8fab9814450d28b4f592dfdb0cd8c2267f5fe8d7043a907d74c9a720713cb056","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252533153861632","occurred_at":"2026-09-16T15:56:39Z","signature_id":"d2849a0a17cc4329034ec17d0d9ec1983dfbaf5324ca580cd9753b318ae0a9bc","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"51205dcbc010a62ffa53522b05b81b83d3e49f95a801c1ca50c89cf5aa533563","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"7b40ea74a666a71f36b6120544e0bab8b70aaea87e5e9b851b3d3cee913ad9be","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"8553584f5882f3e24dad240bf62c9d51df1d078a5dc179dc15818be725a00b81","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"9bdd91449d29da7a6a1036ded0dff71e7cee7dca6d3b72ee155f7272dbc7ec9e","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"f0391e98a8d1c457ea51b54726e3c7af785fdf41b5e338fe271c88dbc805d911","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"52b66be1d91cf6aa4302f446e0cb6737cf15f6d2f47af03effef5a56fd561fbd","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"53f4dbcf0527ee85fd51e8c2f839babdd74e956b703ba4e02f72dc66ca32e8b9","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"593263a65b95e6f760aa7210a803a4f621c8333076597839a59808dbbf66d1f5","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"92ab9e016befa8294f948260d25c72e7a1d80889506782e1f2a4bb4fd849697e","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"de1bc95acf9d163b0a4c5d87ba6c29060b3bb4b653bb97edd817c695a7ef262b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"eb1186f89f1efbd9f2fb8485a07c81a2445ab57e2c0781fc90a1717593153377","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100228663373991936","occurred_at":"2026-09-16T14:21:48Z","signature_id":"27221439d7603c27de714a5d301e36e3ab9f94a39958b4fd4679cc00d417d51d","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100227428122103808","occurred_at":"2026-09-16T14:16:53Z","signature_id":"0240604425a3af254fe1ce6e815c04f07285897c6355ead3254c138853f560f1","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"416d3122655e04bc6a1e6cd87fa70c5433f7bca4d0212f66aecf79af52b1d0a5","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"42ea74c98dd160984e4c829f799e27d45bf2b5018f73c2238f76e9d8d6a3f703","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"4634c0871869430762696feaab7758bce3c5d6614cf0816c70b2c3b8b4d3ac0d","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"773504a23a339bedaebd473b20c28b68aef28fa8120eb503e3c2ada5990e4cf7","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"97a8d855655aa481b47156e39fc63195c1bf1486f4d7c41cbeb84f0001f75fe9","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"d1ada8baec3cfc5a8db7ecb4d82edd81b7f41fd13101da84520ce2bb5cd103d1","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"3b8fdb12977219a6ee2876ab164d9d979ba1367fe5e61c35ac3016ab87e10d51","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100213017273700352","occurred_at":"2026-09-16T13:19:37Z","signature_id":"1e90b34a567db73b55b0b0389a5bd30033829227bcd0152b9cc1ea0cbbb4ce6a","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100213017273700352","occurred_at":"2026-09-16T13:19:37Z","signature_id":"212391e614ece2195740bc51838047e93b2e7ba13e0b53f412f14fba63b492c3","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100196783199096832","occurred_at":"2026-09-16T12:15:07Z","signature_id":"c6e0c1f0e5cc2283be488e36d90c53cb7425625d072448ae938150a9f6555ec0","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100196783199096832","occurred_at":"2026-09-16T12:15:07Z","signature_id":"d49dc00210e1aa19acac371c4b3bbf0262ff2f7023f27f607ca12332c134c5be","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"3bf8a07ef9de6057d5469a6d5677df4989f7484e06e3a81c4e3df9504d3c4305","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"3ec1682204cd21163b91258f7fd79ed3ef2df2235e522890ff723136f72137a4","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"5d3daeaf2541170cf2048425db2ad55ef8977c82724f199e4513a26eb43628d7","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"cedab3b9f607c3ec7848c570f240cf2b35352afbc688b78a3a96a96f564deb7f","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","signature_id":"40e2d334636405f418ffb52c5061fbf30366656d327d9d8fc83b6e9d9afd9837","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","signature_id":"5b34bf27b7432e062ab59dae95c6e2c2ce5726c184df3b6d5751452cf7c1aac8","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","signature_id":"7fe3931d094682c8af6a0a3a715b07aac46854b46c2bbb9f6ad09b29f24ea4b6","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","signature_id":"c5a888e0079b8b0888371862a2ca292117eb1766fc5723aec9962ec739421dc6","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161384225443840","occurred_at":"2026-09-16T09:54:27Z","signature_id":"3fd557fa1356d64ec7477ff53929400119de4eda55d74d194fe4734c675f465b","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161384225443840","occurred_at":"2026-09-16T09:54:27Z","signature_id":"49e9dc4352795e52ee09109b3c354fc6854bce4fbeed1fea1be113f6a6209aa5","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100153419804184576","occurred_at":"2026-09-16T09:22:48Z","signature_id":"1df8582823fde891257155971a259af217e7163cb77525129dce1bd9c178bb48","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","signature_id":"5a06bfe3d19814f7c0a59d9e13f30fd4c6f1a6a7313da746fc7cd3c44a2f7939","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","signature_id":"5aa313310cb85b534b6ae9ece7e869257889473057f8f6e20312e1cf45aa795f","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","signature_id":"8fc547e896d5e55d734d3ddd67ff9ad51e9b0658292885c96f9396cffb7b85a2","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","signature_id":"b19cd1523bca64c96d563324ee6475a23c1199cf8856040f6afdad3121dbc86f","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","signature_id":"d15fe82d34504ca3d38710b46e1d54bc220f9fcfedc628bca6bbc3cb0cd4dd6a","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"3f36db93067dda2082a5ba2aa4539a20530e919b951302455f99df670738d8ff","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"94776e25785ac46d0a71172bc6722ed5b40d2bea8cb9370d89cdf09f1fa5c805","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"ab25cdf796ca0e23837f6fcc81a18f44c0e9d3bfb187f38beeb217788731b679","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"1f776ef61b45ee091fed66db65a127929ae7ae7228c6ef8a3304e727c23cbe5d","pr_number":6879,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"9f89db0ec4b8c3901dcf3ea1dec0354567233659a0ca459f22959ae94e89dbca","pr_number":6879,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100119365197238272","occurred_at":"2026-09-16T07:07:29Z","signature_id":"8485b82a0c9f334d47923eba74cfc4cebce6914810909ffc70f3acbe8579b2bf","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100119365197238272","occurred_at":"2026-09-16T07:07:29Z","signature_id":"f6b4ca993a39a3f8a63f6cce0ebaa94328998c8c08a83ab0da84bb108e61207b","pr_number":6939,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":76,"runs_affected":76,"failed_runs":76,"run_impact_percent":10.41095890410959,"anchor_occurrences":76,"after_last_push_count":35,"trend_counts":[0,0,0,0,0,30,31,15],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":76,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":76}],"full_error_samples":["alert fired 13 time(s) (1 unknown, 12 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T20:11:59Z\n  Ended:       2026-09-18T20:26:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6109056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stkk28pfi15i802bl7aop2360tp9eph-a9i6s2l5j1h0r1s\", pod=\"router-8444b6f56d-w6qnw\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2stkk28pfi15i802bl7aop2360tp9eph-a9i6s2l5j1h0r1s/router-8444b6f56d-w6qnw has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T17:37:21Z\n  Ended:       2026-09-18T18:05:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9556480-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stibp3tbntp3l84dd3nd1p1jo0f3edi-private-ingress\", pod=\"router-8495fb9c6b-n57p2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2stibp3tbntp3l84dd3nd1p1jo0f3edi-private-ingress/router-8495fb9c6b-n57p2 has been in a non-ready state for longer than 5 minutes.","alert fired 3 time(s) (2 unknown, 1 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T13:05:51Z\n  Ended:       2026-09-18T13:34:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2056320-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2steckt12k3f6paulmdgtiackkj5k84r-j7s3j8r2u9z3h1j\", pod=\"router-86c7fcbd7d-pjgqb\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2steckt12k3f6paulmdgtiackkj5k84r-j7s3j8r2u9z3h1j/router-86c7fcbd7d-pjgqb has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-18T13:07:43Z\n  Ended:       2026-09-18T13:35:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2056320-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stedamranrq9pqrbs02e1rt645clovm-cilium-cluster\", pod=\"router-7c5f44b8f5-2rxm6\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2stedamranrq9pqrbs02e1rt645clovm-cilium-cluster/router-7c5f44b8f5-2rxm6 has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023746776109056","occurred_at":"2026-09-18T19:01:10Z","signature_id":"4beb77cb7f662a3ba0c82df6677c24959b741d146512bce1d7d72b33bd6e9a0e","pr_number":7010,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","signature_id":"adf91e76f1e8a11039895567a05fb0e5c3f1d56a9565f8cf0e71b7f1f6e5d0ce","pr_number":7010,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"991b7eb9b2760771a64ba10515aeb5d1b17b39f0d0d776abbc32c3f1fb85abb3","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100887669335658496","occurred_at":"2026-09-18T10:00:26Z","signature_id":"c3bce50243ff744e90a9975f976bf7851614b19682c44e007ffa00d7c6f101f9","pr_number":6999,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100886446859620352","occurred_at":"2026-09-18T09:55:36Z","signature_id":"fcdb5ce6d5eb5426a483be7faf7e19f591fbf253b3ebf7b8ca13d6e2fd0d11f4","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100885599773790208","occurred_at":"2026-09-18T09:52:14Z","signature_id":"002a8aff34d1219bae02653dd6f6335a032c8c53538dc0e88e47e06a4653de19","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100877868748771328","occurred_at":"2026-09-18T09:21:30Z","signature_id":"7eefe8fce4efeddab6a7bddfd5f7e5cb50289671f56d6e25708c25b9305ce796","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","signature_id":"be273c1646946db878b565bca63154b691913899e238a33aee949d4d46ae8e4f","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100867050745368576","occurred_at":"2026-09-18T08:38:28Z","signature_id":"d6db3f97bdaa9f3b8874f62b7846be7bdd694b5e229a6dfe93f7bd212a0dcb02","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","signature_id":"f47f0b6edf615e207ea20e3174c9c4b6c3860a8cd322039375d46696bfbe5705","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841559414018048","occurred_at":"2026-09-18T06:57:14Z","signature_id":"7b986d6b9e982cdf65d80f902e9563f9d1125a3f54f3c1a12116db416ae5f10d","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827581187100672","occurred_at":"2026-09-18T06:01:39Z","signature_id":"56136cc3eeacd2663934d686bb2d9b29fb8b76aaa1ea7443d05087ccb6f12b10","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100812371122458624","occurred_at":"2026-09-18T05:01:15Z","signature_id":"e85df63033962c2849bfc318fc85b0a8cba9948dc2466a5fa45b6b2724994bfb","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","signature_id":"4750aa456ebf384aa187370c588c7897567d97246241f2ecee3733d106d2cd85","pr_number":6984,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780897212043264","occurred_at":"2026-09-18T02:56:11Z","signature_id":"a0cd1baa14d1b2c8e6eaa3868608b05d0234717e1c141c4d01a43194b1743c96","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721105009381376","occurred_at":"2026-09-17T22:58:35Z","signature_id":"82e0a12c7cd95fd907f63543eeac97bdeae8d5f9e5a424bdde230db4ef5d4c57","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"1033d58df80ee9a7d767bdc19425ad417e226db41f4f4ad1a7aa6ed4d2688d8a","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100699515785842688","occurred_at":"2026-09-17T21:32:45Z","signature_id":"18121697c147e2ac50157b93c7f4175f042996dac56448b883bc040150ef3bb5","pr_number":7002,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"d2aa839fc01483bc5381babfe51639f5c2325311fddfd8c5d39caa32de46dc09","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6998/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685757428535296","occurred_at":"2026-09-17T20:38:07Z","signature_id":"927ffe3fc5118bce586a8a07b93ed80685d7fe4d247724a208069edabb849e6f","pr_number":6998,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569759195074560","occurred_at":"2026-09-17T12:57:11Z","signature_id":"2b92ea2834b8405b970ceee37d04d32b047f88649536fb5e87c255a7cf6b9715","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100566804978995200","occurred_at":"2026-09-17T12:45:27Z","signature_id":"4815c5ee092aaeb9f17a4f179de41c68c528994add0634df56951333903777fc","pr_number":6886,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100563400982532096","occurred_at":"2026-09-17T12:31:48Z","signature_id":"ece77cd8e3e47a8b500cc23dba72bce96033933da8cfc561a7bfe5d93a950f77","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"e8c855a1e7025053c74a73b5e48c4e6e96debc9ba1c7d6b424796483466cdf6f","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"777817b1e387b1f4492402d2f5429c721df7c24785215c5257845216b59cfccd","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6994/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100557383175507968","occurred_at":"2026-09-17T12:07:52Z","signature_id":"b66eb0c3d6894149e8e5bee7c3f68c812916f17da4df7663f06be67bb5aedc35","pr_number":6994,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100556682735128576","occurred_at":"2026-09-17T12:05:14Z","signature_id":"8d0248f7d9c1bf72afe316ba52f0dec7aa9b6c443c3506cf8e8d1a57be0d1c1b","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","signature_id":"f10ddf6a0df1f5dbb5292b3e49c11fae37f0c18b3ec87fcd7444bc0f39779449","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554684329627648","occurred_at":"2026-09-17T11:56:52Z","signature_id":"e2639a0797c1c9a4a16eba734d19e610cbd443c3f33dd4bdd37f3d91b31ccf54","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"35bdd9c6de24d9640b64ae41e61a7affc59db651a49ce7a2fb22727cdcad6dc9","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100525488945500160","occurred_at":"2026-09-17T10:01:10Z","signature_id":"dff627a9d4028dda60dcb2cc608e3aeb26b1fb64501996026bbb8ff831d19570","pr_number":6972,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524896286150656","occurred_at":"2026-09-17T09:58:55Z","signature_id":"fd2143a50a59cd5afd961a3854a9a94e4c9bfcd849575bf08cd24c269273e2c6","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","signature_id":"39a62cd962057ec142de0e390236317a312acabc0d41e0544a266cdbecf8c0b4","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100523023395196928","occurred_at":"2026-09-17T09:51:23Z","signature_id":"360514ce5b258c6295b00ba9d2f1d4fd30911299b6a3d2dd1a60f05cf773940e","pr_number":5724,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100520057816748032","occurred_at":"2026-09-17T09:39:42Z","signature_id":"5472bf78238dce0fe2f94f0aff922f667d82ac0c02290e6e2cd57f8d35e3a169","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"a26f8c2c335980bba3f6e37f89e2b89d15ca7b79b6682f883c830ab2918e5b05","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100507818527297536","occurred_at":"2026-09-17T08:51:04Z","signature_id":"7e105769b349ece072f3c06670207d3d4017ec8870c2a43a5330028ceb5d52e3","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506297609752576","occurred_at":"2026-09-17T08:45:01Z","signature_id":"962f9008502054736ab6ef9e552851e993e6894bbd6022a1a131a064d8689aee","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495749199433728","occurred_at":"2026-09-17T08:03:06Z","signature_id":"d549df554d86a3fb7f1af2edc775ca43f25de1f5239534d866f56be388d7b3fb","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494074514509824","occurred_at":"2026-09-17T07:56:27Z","signature_id":"371eb10e8c7a8df40477f826f6a0c01232597b190d53940fdbe6a8b4c627ede1","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494069930135552","occurred_at":"2026-09-17T07:56:26Z","signature_id":"f82050e70a810fdc9a71c82fe836c8741b43edf42cceee0a76e6119d660eb530","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488796448493568","occurred_at":"2026-09-17T07:35:28Z","signature_id":"826198aaa6fd1e4924bf9a5846057c446813d101971e8e7ab1f42a73a8fd3ad8","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100463746039156736","occurred_at":"2026-09-17T05:55:56Z","signature_id":"b8571782c9062ff08f362067402808930639b058a5f971bedc267ccc71e53617","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"082cd49a16ae701dbb38b1b79f0d1ac12e5a9df37271b1daf757d49fb6b996f0","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","signature_id":"3ddbbbbfcbe35abb46fc03a012581caf1092f9cbcef18f864c7ff0cb4097904d","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100385816462233600","occurred_at":"2026-09-17T00:46:16Z","signature_id":"e6916222b7c43fadfb77f5030602e037edec83d0ef773a05b13d08de9f0efc19","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100354655547559936","occurred_at":"2026-09-16T22:42:23Z","signature_id":"b66349955715ed443f1362f9a218926c741bbba15a7053b8231a572f7bce0c04","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351996258160640","occurred_at":"2026-09-16T22:31:53Z","signature_id":"e2e746b818d8319297b0cfc1ce06bb87bc7386a6e269943bf13e78c9d13237ab","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351264033345536","occurred_at":"2026-09-16T22:22:06Z","signature_id":"db6b8a56aaf730fdd696fc547f30dac94889302e62174f2dd2508fcd332898b0","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100344321164185600","occurred_at":"2026-09-16T22:01:23Z","signature_id":"600a7515642ad65bbfc931d4dd2a57d1a3c95b9594a86121e772644509ea9e97","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100335879556960256","occurred_at":"2026-09-16T21:27:50Z","signature_id":"c599c77a6372b3affcde70e85dd51dd5124ab9de3cabd625edc84ba8a133bbfb","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100327783220645888","occurred_at":"2026-09-16T20:55:40Z","signature_id":"4c70e8e6dfdf51fe895bb78044f498f38b34d7b31ddb973e4ed72258e72ee568","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"d21baacf92b823fbe47d168aff240c5f90998a22314dd5cc365610b36ceba21b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"ee56a79fd88c61821c015922fd0060df48fa8ea6f29e86e41766eb43042fab32","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"d8457d1570ea90a26bcd98437f37670575aae28bffe0f986f9b004d2d9264040","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298749917007872","occurred_at":"2026-09-16T19:00:16Z","signature_id":"4fe55ebf2e3687f10b470f9b50ef984b7f154b6ecd6235b377b7abf35f5fe025","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298495381475328","occurred_at":"2026-09-16T18:59:17Z","signature_id":"ed3bf953568238e27f1b875c5f5f12b7c39936c7b777f8037f83dc0b1f8c2feb","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","signature_id":"e4b9c749f68fadead7aeb206b02a1f85e9ae2c2c7142b04d727f3089de1f8541","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100262131709513728","occurred_at":"2026-09-16T16:34:47Z","signature_id":"b7c6ba500fe6358c309a0b082682885bdc0c4016fb309f71ff96ef107142bcca","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"8a55cdb17b2e0960799afe76d087110a5ecf992561f62b085684e8010de63d22","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"d167cd058619d1445e1dc6c6ed93bc2e291c8bbbd9eba4b08df3c14423f77517","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"4778f344e9b25530d8b9156641a99f6b2838c230f7532738e3f69e081b9f7a7e","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"045a81b731fe6b760898ced74dc07c1acf462fd8638ae785f456d496a795842e","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100213017273700352","occurred_at":"2026-09-16T13:19:37Z","signature_id":"422af2d2e106cf074701d11ee92eac11656adc2f791fd92e942ac51d5ff857ef","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","signature_id":"aede2952cb21cd0c07ff4ac502b4109b4236290ff6f43397958c99820cf77924","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","signature_id":"3a6cb6146cbf8c095d871f7807f59ec635628a6cc72ba515bf71641eb6ad5bd1","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"7451ba3db591daddff6044c850b50a806f031b44b26c0a9cb44f8b82e28dd244","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","signature_id":"382d6e1a50a58344a999c679c7ef69f1f3c041161e556f0be9d14b621d84b09e","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100176649516486656","occurred_at":"2026-09-16T10:54:59Z","signature_id":"f201db6f99c06ca51a3a0a3934aae0846a6ea6ba393747b6f6edc6872fb99a72","pr_number":6952,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161384225443840","occurred_at":"2026-09-16T09:54:27Z","signature_id":"ae8593e2f93d516941e79280e7827e07a691be6004452673f6399cae8b22f618","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100153419804184576","occurred_at":"2026-09-16T09:22:48Z","signature_id":"3a248c85aeccd44a74b5e82dc19ae5115320308561bbe219e225440971c85f86","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"cf50089acb0846a23f15800cf9d1588e5a42034409429bcfeb7f8b4bdce19e86","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"74727d2f774b690907a06926c56a326fbc39e84388995b5a3c7eceec1a64f678","pr_number":6879,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6962/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132023019507712","occurred_at":"2026-09-16T07:57:47Z","signature_id":"22a7d0041fe9eca9fd2523ee31fb7da254fed5da06967c558019791f8f2e970a","pr_number":6962,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"05b02974c8e5aa95fe3326e72aadd1d9c24fabbae417684be2c2a88ea3d78009","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100124639266082816","occurred_at":"2026-09-16T07:28:27Z","signature_id":"1bb0a319d54a133be2e86c848056c08984c613c97a376359e9e2876ee27aa821","pr_number":6856,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":67,"runs_affected":67,"failed_runs":67,"run_impact_percent":9.178082191780822,"anchor_occurrences":67,"after_last_push_count":37,"trend_counts":[0,0,0,0,0,46,21,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":67,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":67}],"full_error_samples":["alert fired 7 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:00:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8t9zj\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-8t9zj (namespace arobit) on cluster ci01-j3118336-mgmt-1 has averaged 131.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:10:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-g6d4v\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-g6d4v (namespace arobit) on cluster ci01-j3118336-mgmt-2 has averaged 183.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:04:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7b547f7d8b-bz92h\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container mgmt-agent-controller in pod mgmt-agent-7b547f7d8b-bz92h (namespace mgmt-agent) on cluster ci01-j3118336-mgmt-1 has averaged 183.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:04:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-86c78bc956-rx5ct\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-86c78bc956-rx5ct (namespace clusters-service) on cluster ci01-j3118336-svc has averaged 829.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:08:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-vnkdq\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-vnkdq (namespace arobit) on cluster ci01-j3118336-mgmt-1 has averaged 166.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:09:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7b547f7d8b-pnvkw\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container mgmt-agent-controller in pod mgmt-agent-7b547f7d8b-pnvkw (namespace mgmt-agent) on cluster ci01-j3118336-mgmt-2 has averaged 166.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-17T08:59:26Z\n  Ended:       2026-09-17T09:17:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-86c78bc956-ljk5z\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-86c78bc956-ljk5z (namespace clusters-service) on cluster ci01-j3118336-svc has averaged 226.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","alert fired 3 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T08:17:39Z\n  Ended:       2026-09-17T08:27:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3085056-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-49zq5\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-49zq5 (namespace arobit) on cluster ci01-j3085056-svc has averaged 168.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T08:17:39Z\n  Ended:       2026-09-17T08:27:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3085056-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-d4b6fd9f9-p4w2z\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container aro-hcp-backend in pod aro-hcp-backend-d4b6fd9f9-p4w2z (namespace aro-hcp) on cluster ci01-j3085056-svc has averaged 377.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T08:17:39Z\n  Ended:       2026-09-17T08:27:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3085056-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-78587f7f-wtrr7\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-78587f7f-wtrr7 (namespace clusters-service) on cluster ci01-j3085056-svc has averaged 226.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","alert fired 4 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T07:47:54Z\n  Ended:       2026-09-17T07:53:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6077952-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-648575ffd8-pgfs2\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-648575ffd8-pgfs2 (namespace clusters-service) on cluster ci01-j6077952-svc has averaged 276.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T07:52:53Z\n  Ended:       2026-09-17T08:00:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6077952-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-x8286\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-x8286 (namespace arobit) on cluster ci01-j6077952-mgmt-1 has averaged 128.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T07:54:53Z\n  Ended:       2026-09-17T08:01:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6077952-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-p74sm\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-p74sm (namespace arobit) on cluster ci01-j6077952-mgmt-1 has averaged 122.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T08:17:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6077952-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-648575ffd8-526ht\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-648575ffd8-526ht (namespace clusters-service) on cluster ci01-j6077952-svc has averaged 185.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100485495183118336","occurred_at":"2026-09-17T07:22:21Z","signature_id":"f62c0a71de4ea91a4231cade3331db31d540a634ddef3ee99f2bb1c6a99266f1","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100479104343085056","occurred_at":"2026-09-17T06:56:58Z","signature_id":"aac5573ba90ecce2e869b424d24d0b2a557bdaf7fb68a59bdb1ca60dcfc91361","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100474592526077952","occurred_at":"2026-09-17T06:39:02Z","signature_id":"aa91dd60d940dac3239718e70007e00a82448bdb09061df222f973e65da4b8f6","pr_number":6952,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100470565717413888","occurred_at":"2026-09-17T06:23:02Z","signature_id":"036570c8237e20bff362fa84f18b2757ecf6b7478b115ced0b1b6d96b52ae4ba","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100470315283910656","occurred_at":"2026-09-17T06:22:02Z","signature_id":"34004eeb935c0bb4ce4ba308740f33575b771c563e1c3c5cbd660527dc370ef4","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100469063225446400","occurred_at":"2026-09-17T06:17:04Z","signature_id":"07f59a2e15e5179fe3fd25749784f754d222157d8b27384b9e1cd64842ae8768","pr_number":6986,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100463746039156736","occurred_at":"2026-09-17T05:55:56Z","signature_id":"aaa081b0ec7f2d8d4a7e0e90f29ef2c663585ebb663aa2ffcb4b659e94e0c5f2","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100445405488615424","occurred_at":"2026-09-17T04:43:03Z","signature_id":"98874527ffbf8450aeb4dca36d3c24d835d1bfc8669e736b7a62e3aeb588719f","pr_number":6952,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100440653564481536","occurred_at":"2026-09-17T04:24:10Z","signature_id":"98473ea7cdc506ea6ed06c90a329992858dfe045473eb058bcb45937743f020e","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100438339525021696","occurred_at":"2026-09-17T04:14:58Z","signature_id":"d43422fbdada1a940ecd84da27859063da487a9ed357cd0dce3b03a5e1093c47","pr_number":6986,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100435900084260864","occurred_at":"2026-09-17T04:05:17Z","signature_id":"a15a05f0f8abcad7e64ec9115ec84f6ac3e540a459d0e285796dd856a7a6caa3","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100434265043898368","occurred_at":"2026-09-17T03:58:47Z","signature_id":"3e9ecaaa55541e660e55a54f116bbcb93077443ad73a39702bb7f87aa56a43f5","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100420606720741376","occurred_at":"2026-09-17T03:04:31Z","signature_id":"8469f16144ba929df5b9d7bb8ddc370454c226da3349d92d91c2c32c02a33ba7","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100415215534346240","occurred_at":"2026-09-17T02:43:05Z","signature_id":"05d61646d01952c54a7da4864113e0532f1dbe45108ad4402418ae2eaaf5a2b6","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100404999925272576","occurred_at":"2026-09-17T02:02:30Z","signature_id":"030419f084e0ac9f8645ed83bbb93e54a5eee29ea545c7c537285ba77f7a19f7","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"09d8e2ae3fbdc5cec5986413598d96675056845bffdde0b6a135d4a6c2fcafa1","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100399662610518016","occurred_at":"2026-09-17T01:41:17Z","signature_id":"5a274f8276581e7a1744e1e3283d6f882212c3ec0a22900f40bdbce973372f71","pr_number":6009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"5eff94552133a60d52e39851d75edd9d10579e18460fb3418389d4051065a47d","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","signature_id":"f729f652c9b63e98893c72b91f3a1326eda28d14e829f566594109b7326b49cb","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"eeada1c413c3ca5fc2af1c6cbb216ce484bf15370729d3a7b144bcec22517b3c","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100385816462233600","occurred_at":"2026-09-17T00:46:16Z","signature_id":"a4314857c749da303216d314351149e0065b1d08354cb26b1fbedfc555fc6412","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100373409937494016","occurred_at":"2026-09-16T23:56:58Z","signature_id":"1e3921ef26cf400f3e3eaae679c2c7938f1ecf3228c5ec7fae0e28ef8fa1159f","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100354655547559936","occurred_at":"2026-09-16T22:42:23Z","signature_id":"940a5892f0bde8defb7629a002baedf9228b8f92a2cf58838e303561286c491d","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100353767223332864","occurred_at":"2026-09-16T22:38:55Z","signature_id":"e71be1b08c965527fccf228721850a63e0d7ab82463979308010bd46b7d5afcc","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351996258160640","occurred_at":"2026-09-16T22:31:53Z","signature_id":"fc7a5f2beeb914b7f9d5e98e516ffb3d0e395b2c10277b6a8a41faeeb997a062","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351264033345536","occurred_at":"2026-09-16T22:22:06Z","signature_id":"38cba13ec304a2d65c8ec3a2bb0373cae43840dfeeed1f7a55cc2d8c840f0488","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351223038218240","occurred_at":"2026-09-16T22:21:29Z","signature_id":"fbe220275de66dc82c17bd70379b1968bfca47f26279aaeb942365f7083c7aae","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100346022097063936","occurred_at":"2026-09-16T22:08:08Z","signature_id":"5098cde8f56024f9b94cf40f2a466897158f3f3994b796f3a58cc2d52fcb7504","pr_number":4878,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100344321164185600","occurred_at":"2026-09-16T22:01:23Z","signature_id":"f7080c270925219a311e60803887fccb7900c43032060412377f7a288ab7c269","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100343556559343616","occurred_at":"2026-09-16T21:58:20Z","signature_id":"180e6e60fd74213dba13f27436fee7ff492e29f8f76edd5be5c54b377947d936","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100335879556960256","occurred_at":"2026-09-16T21:27:50Z","signature_id":"71b22a11302a00be21922c71aa067e721f76475093f648235ac53a36e12bcb39","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100335557920952320","occurred_at":"2026-09-16T21:26:33Z","signature_id":"ceaa91f4efdcdb0d1c542d52b327737fcf8ae4ccbf48f0c17aab82acbcc99238","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100331448274784256","occurred_at":"2026-09-16T21:10:14Z","signature_id":"30e6e0d7b43f0480d6e10d35a84534dd048930cbdf424724d15cc0834916efb3","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100328454825185280","occurred_at":"2026-09-16T20:58:20Z","signature_id":"cfb1506b480d229252a933ef91e34400b8d02a4638f13cabb01dc1d4175a17b8","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100327783220645888","occurred_at":"2026-09-16T20:55:40Z","signature_id":"18e89a273ad3f5766b2dd6e52e34589ba49632a0e92e5d7b81de104647e14a35","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"29b55dd4d895c72d0d59b9865df950f870325198fd7e112f1858c956d6c3812a","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"4ef72abc8b9948fd9bead06d17d6310bcbc699657b6ee721fd16b3106ac79918","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100314659356676096","occurred_at":"2026-09-16T20:03:31Z","signature_id":"d4858edb57d6296d8d8b3f66a13945cd04dcb772e32b66ac68791e2b7d64dc15","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"078c270d3b43c225b1670f600e86beb8fac31540b45fed0edb0e8f2b88023a38","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6904/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100310890917138432","occurred_at":"2026-09-16T19:48:32Z","signature_id":"ba7f166d742007eec805103f097b0619e99ae0ecb99c10d11604e7456c1148b3","pr_number":6904,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309676229922816","occurred_at":"2026-09-16T19:43:43Z","signature_id":"d7f945bb6e41f7832c5eef4f84c2ae943bce6eff034300bf30a51c4a754e1cdc","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309343839719424","occurred_at":"2026-09-16T19:42:24Z","signature_id":"efa1057a04dfc40b6e037e249cb0dc21b2c09a49fd06c904c78795387f5897e8","pr_number":6403,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309325879709696","occurred_at":"2026-09-16T19:42:19Z","signature_id":"c35a6da3cb62c984e490fa20921e5205ae8f9b5d5f473a81f2bc4d92f07256ce","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309243423887360","occurred_at":"2026-09-16T19:42:00Z","signature_id":"4e635a5a55fd9c70e99f4fc90a819fcbe842ed4f80ffca35e85d712a75adda03","pr_number":4878,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100299874200195072","occurred_at":"2026-09-16T19:04:46Z","signature_id":"94b33034edb32ac378faab020acb7aa1cb586b97112ae6c5f8f64b5e0f39f1b3","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298749917007872","occurred_at":"2026-09-16T19:00:16Z","signature_id":"569b1e0aed53209c579e2ada482c82175262b769185ccf3a5bafd789625f23b0","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298527354654720","occurred_at":"2026-09-16T18:59:25Z","signature_id":"2515546ca4a2ed9cc66209f7f5a861f329da8544f35e1712e6bef48626b727f0","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298495381475328","occurred_at":"2026-09-16T18:59:17Z","signature_id":"b7bbff7c4bc8e38db57964016b3e5dd22cbf69711555869ce39fed0d057ef5fa","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100294017282478080","occurred_at":"2026-09-16T18:41:29Z","signature_id":"d6b769c42dfc7af863ab56e421478bc2130ed7b8030470653f093468c2ae9445","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100293442402783232","occurred_at":"2026-09-16T18:39:12Z","signature_id":"6d07e5dea3d8417306b6aad1d07607ec094c290a529075bc5ae0f6c507ae8dc7","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100285588732121088","occurred_at":"2026-09-16T18:08:00Z","signature_id":"3165e7829bfab81efada837f8a5a846a713826a1a775685756db96ad2a2e7a0b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100280971763912704","occurred_at":"2026-09-16T17:49:39Z","signature_id":"5241ddfc74d49c5ecad2616742ded97b4a5f9bf5fbeee0f248252a9d33303902","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100273649465430016","occurred_at":"2026-09-16T17:20:33Z","signature_id":"2aaecc7a8aa85a870eddb45d3a454574d39a17a2e1971a571ef303ebfa1b5dfa","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","signature_id":"4aa09e305fda5dad99f7aa547324f1cc9a5313a1ed785f8832369357abebfa2a","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","signature_id":"e80538656acc5e08b78c0da3cbda2f5f642ff686a707fd278f435ee95c791d0f","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100262131709513728","occurred_at":"2026-09-16T16:34:47Z","signature_id":"8bca23390b6e128f806583b6f43999dfe3e0891c3867f9b00f777f7b70551bad","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100260298874163200","occurred_at":"2026-09-16T16:27:30Z","signature_id":"641a1726bac71eedb2360eec9a405c0d1e826983dcd4ea335ba7c7dfa3cebe9d","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100260058020450304","occurred_at":"2026-09-16T16:26:33Z","signature_id":"9a105f0d73b76a92d17a2de5a3542ed76121642b109339640b150bdbd8718601","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100253677548736512","occurred_at":"2026-09-16T16:01:02Z","signature_id":"e26ae0379e9c2f78baac16193a817ee1d4bd6be52f5b22f001aad146994821f2","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100253157446651904","occurred_at":"2026-09-16T15:59:08Z","signature_id":"bb91f870f2d6651db04eb26377a6ab0f560e56353470294354ffb9ea8c0407f9","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252538870697984","occurred_at":"2026-09-16T15:56:40Z","signature_id":"984ec4e1000694cd92d63a32c0f7a6e5d7b7f677e37cfce55ad670237d8b5da0","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252533153861632","occurred_at":"2026-09-16T15:56:39Z","signature_id":"59e4de1f8e513f5cc08bd4e593ae29483ec56ee54e8e92faff2ba25b8713cc09","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"fde97b6bc2d22cd4f0aa0ec9116922422f6896f61d1b4b08310d88dfdf610931","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"5176305f0eeec8bbbaa6ff316c0143a9416bf749550533697343b8a801ca28cd","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6923/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100244030364323840","occurred_at":"2026-09-16T15:22:38Z","signature_id":"3af7515b2be64cbe2958e1955e72cb60cdb6b11e0681c1a634bbc8ed9f8e15d6","pr_number":6923,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100243112008880128","occurred_at":"2026-09-16T15:19:04Z","signature_id":"8cecf1b875c80d5aaad598063b3c383cfb7ef9d6225c392907d6632715156b7b","pr_number":6923,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6726/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100185438990897152","occurred_at":"2026-09-16T11:30:02Z","signature_id":"5d5dbfb417b6897cd24a2f1e6ad614604212fa121b9f81e032502d4c16e94ed3","pr_number":6726,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":42,"runs_affected":13,"failed_runs":13,"run_impact_percent":1.7808219178082192,"anchor_occurrences":42,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,22,19,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":13,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20260901preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001520108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-ptc9sq59cf2d\" to finish deleting: GET https://rp.j6053632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3852ce7c-96ee-4d91-9970-a39897a9c07a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3852ce7c-96ee-4d91-9970-a39897a9c07a\",\n      \"name\": \"3852ce7c-96ee-4d91-9970-a39897a9c07a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T14:15:14.231768617Z\",\n      \"endTime\": \"2026-09-18T14:39:19.156773648Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2stfc33bsiqt6ka6nm1u076ksjbrgtab still exists (deletion dispatched at 2026-09-18T14:15:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001520108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-ptc9sq59cf2d\" to finish deleting: GET https://rp.j6053632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3852ce7c-96ee-4d91-9970-a39897a9c07a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3852ce7c-96ee-4d91-9970-a39897a9c07a\",\n      \"name\": \"3852ce7c-96ee-4d91-9970-a39897a9c07a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T14:15:14.231768617Z\",\n      \"endTime\": \"2026-09-18T14:39:19.156773648Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2stfc33bsiqt6ka6nm1u076ksjbrgtab still exists (deletion dispatched at 2026-09-18T14:15:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001226768\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-jwdh6h\" in resourcegroup=\"rg-candidate-4-22-9rtkhgwpwkdp\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/497b563d-6a90-4107-8ea2-18e801b64cd8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/497b563d-6a90-4107-8ea2-18e801b64cd8\",\n      \"name\": \"497b563d-6a90-4107-8ea2-18e801b64cd8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:09:44.365139487Z\",\n      \"endTime\": \"2026-09-17T13:33:50.717397424Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7gelb6kbkk61h7dh9rg6bgds1mn2 still exists (deletion dispatched at 2026-09-17T13:09:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001226768\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-jwdh6h\" in resourcegroup=\"rg-candidate-4-22-9rtkhgwpwkdp\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/497b563d-6a90-4107-8ea2-18e801b64cd8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/497b563d-6a90-4107-8ea2-18e801b64cd8\",\n      \"name\": \"497b563d-6a90-4107-8ea2-18e801b64cd8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:09:44.365139487Z\",\n      \"endTime\": \"2026-09-17T13:33:50.717397424Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7gelb6kbkk61h7dh9rg6bgds1mn2 still exists (deletion dispatched at 2026-09-17T13:09:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001790120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-8928t8c2w987\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7f319ad2-39f3-41e7-b4bb-2445dadbddeb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7f319ad2-39f3-41e7-b4bb-2445dadbddeb\",\n      \"name\": \"7f319ad2-39f3-41e7-b4bb-2445dadbddeb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:04:21.545838494Z\",\n      \"endTime\": \"2026-09-17T13:28:30.638150402Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp95g2hh8pms8h7312ekq6dl4tr17v still exists (deletion dispatched at 2026-09-17T13:04:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001790120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-8928t8c2w987\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7f319ad2-39f3-41e7-b4bb-2445dadbddeb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7f319ad2-39f3-41e7-b4bb-2445dadbddeb\",\n      \"name\": \"7f319ad2-39f3-41e7-b4bb-2445dadbddeb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:04:21.545838494Z\",\n      \"endTime\": \"2026-09-17T13:28:30.638150402Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp95g2hh8pms8h7312ekq6dl4tr17v still exists (deletion dispatched at 2026-09-17T13:04:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100931988906053632","occurred_at":"2026-09-18T12:56:34Z","signature_id":"946ebd2cdc29e94011829e7ef21165a52d1801fd362344870c8d6a3e7c5269c4","pr_number":6999,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"1200ec6f2e462f6a4057f0b6736f389e0eb4918e85be52611ba1988ed4a6b80f","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"26a26ee01c1fe63e9854b87ab08eead6a6fc7c9019b032530af1bdbbab01d85b","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"36c7d0ac2608fdd9b180eb76cfcef49dd063a5ddca7a5e1dd0eb82655e85e2bc","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"397c03aebb8f9974cabd4c075afe35800b5602c55edd622b828e4ee997bc99ca","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"51e487e0e9fc1b87185142de83749a342d6962b63a6106533902cf6918e54d5e","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"54d2d9e8120866d9194ddacd546d77becc75e8a251ac250bfb9fb8db3e040d85","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"83cf0cf00f201ed62094249bde1cad4326fd870d9fd95f573466f70f995ca9f3","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"a7c143432b9aa1db66f17c395896aa0a31866970f3d2152b89df2ba59a93d938","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"d5580f5abed7b32fb1241b3c5a25421aa30df617d15c7a847acda8ee9363e8f9","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"f1c4779490fb4bdafbfc35cb730136fb51e8d2baae0539cf8ed6ed1617f9f5b5","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"f1d1ec88637d584406805321e304585733a1d0d5f1dc372e90627e8f47025e62","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"1d5a773360bc54998f805181631ae95d9d4f7e700834cf8b2a7758ff8a01a8b3","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"5ffc110d49d5553b43d54e9686ac2a42d55e70924452ef37b6b6d552874166f9","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"8f0ccca9b1ab0f62cbe8877698c2eb88ab3fdc916c573bc80063829aa9d361db","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"d478c40d48df905f6ee8e97dc60c2a82d7080ab4ce2d7fdcba0c3903ce1b257b","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"f1fb92d8ffbf4155d66435595ff699141fa92d0cac0f75c60e648373d685e73b","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100553657568202752","occurred_at":"2026-09-17T11:53:12Z","signature_id":"e9fcf1ca8ec5a4e5b455439c42c7c5bdd9023eaf9099b980ad11a8e344c2334d","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"554758499da2dbd5ca2e6de15b8c5c7a3ee81c45e3edf3c5d71f98a2d63ec2c6","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"ff7aa9ff9517f007abe64da89b86cee0dfefa327cb83d202da998d1607130a76","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100314659356676096","occurred_at":"2026-09-16T20:03:31Z","signature_id":"06e7cdde30cfbab4250217a65b741f57948c596ead351300b63debe1170a60a3","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"62f925cd8f107a148e561a08817704dd4da0dc08360a885c312c5776ffe6894d","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"e259668ae1165ff12f8afd3ff5be66f96028094b8d861c7cbdff5a5c0c31f019","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"460210ebabcf024c04114b174b6545f46b58b1c28d70303d86e3b1dce093c204","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"a552e8119646d9c743aa9a252024bde026b39e6cefa5828c2a635099d2355f2a","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"c81fbcddae76e3d46ce74bd7eeec6dbb65a70aa3f34997838af6880b8bc4b471","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"f38603c674107979a7e5113b087deeb85d932689f0a71354d02c3879c176e60d","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"09330ba162b6af089c34735c70eca9ddd91f6516659badb8a358d7c06fa7a1f1","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"4baa98b743ff58863c2c3f11ae6c7794d41c41f07100fe61766b0978f6746f19","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"6b6d7951973760ce40ada8345eab0fc3affb480bfb3af182b9b92a268d16b16b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"cf0c7c2f7324da55d7b02b3480eb3ca7ec6d10efdd79ac18d6876f5a4e4fc8b8","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"32832ce99401d97a958b7dcdfe20140118a267411f3e842c201974409fd354fb","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"47db8067bcef5f852d6be174031d8a4891925f676b5063754561044e04b33b85","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"6422616296d3a5a4112b796133cccebf1e701aef5be2522f1440d40d7f73f94f","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"e114928abafbdcce9fc1b5a0f9590defe570e1bf5ceed7138ffdb8bb94282914","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"e94f3f5e69e8e136647e4790e270a1391748aa6703a074a6fa38b2687beb9db4","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"ea4b0a4eb2c285656f848886af6bb6a27818c286a7b34ef24973fc4e223cf7a2","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222891458039808","occurred_at":"2026-09-16T13:58:52Z","signature_id":"1cb320d43390d047edfe367eca720f9ae752f792b2067b9a7d668b21e611b437","pr_number":6952,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"16a76a4ed4fe90d47f48a9657d0216e8d1f87acf39a8dad4d8ba5c30878fe100","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100188464501755904","occurred_at":"2026-09-16T11:42:04Z","signature_id":"139ea4eb9a07c176246f76b251911c3691a6cbbed5076e752b52b4042c281baf","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100188464501755904","occurred_at":"2026-09-16T11:42:04Z","signature_id":"85767451e4aa0503e59ddd278738b04f06b742245943e9225dbc7e3560e04b05","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100188464501755904","occurred_at":"2026-09-16T11:42:04Z","signature_id":"d4cf5fc8868b958b193d13f740ed514c1bae0dd9b8489c2b78271801c3f259d7","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":33,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.273972602739726,"anchor_occurrences":33,"after_last_push_count":26,"trend_counts":[0,0,0,0,0,7,26,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110a440\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qr9h99g92762\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2e04524-dc8f-4310-a549-fb6dc0199f5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2e04524-dc8f-4310-a549-fb6dc0199f5f\",\n      \"name\": \"c2e04524-dc8f-4310-a549-fb6dc0199f5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:28.114427313Z\",\n      \"endTime\": \"2026-09-17T02:31:33.96482384Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110a440\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qr9h99g92762\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2e04524-dc8f-4310-a549-fb6dc0199f5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2e04524-dc8f-4310-a549-fb6dc0199f5f\",\n      \"name\": \"c2e04524-dc8f-4310-a549-fb6dc0199f5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:28.114427313Z\",\n      \"endTime\": \"2026-09-17T02:31:33.96482384Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0aba0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-ms9kkl5qh4rm\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\",\n      \"name\": \"7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:38.567378599Z\",\n      \"endTime\": \"2026-09-17T02:31:43.962353158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0aba0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-ms9kkl5qh4rm\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\",\n      \"name\": \"7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:38.567378599Z\",\n      \"endTime\": \"2026-09-17T02:31:43.962353158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cdc740\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9lhfrbsqkhjk\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\",\n      \"name\": \"39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:27.808727421Z\",\n      \"endTime\": \"2026-09-17T02:31:33.989605445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cdc740\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9lhfrbsqkhjk\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\",\n      \"name\": \"39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:27.808727421Z\",\n      \"endTime\": \"2026-09-17T02:31:33.989605445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"02219053511d8a21950017349ee31494aa267bbe162808b309c406afe57bd88e","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"0ea6e92c609e741767496ed0bca205dd97df87334b05401151ca3d9e4297e4cc","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"139bcc210692fb4c4608f88c1e62ffbae5cc26a9ea38e8a550f86dc64be5bb37","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"2161ec92ec6ee1b1f4126d8043c75c164aa59ccae1d75ae8fdecc279478acefb","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"24e450fae1afaf3835ed114031825dba965e2638d9e2a4b78361d363bc7d139c","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"284fd36dc5ef5d74feffb6a93226bcfff835efdad8f1f85bdf032a8053cc1b6a","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"2e37c10dee44a98ad4c5d8e21f945e822bc6d1d157c0aec08182d653353bf301","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"2f9108fe18776541cbc85ef378b8f266052150ee76b5c630519c31e933ddb5b9","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"39e9738b3570d1ad49126bbe45de1c14c1222d36b148883b549374de80d01c71","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"417a674aaec0692852dbca6d977cc89fe23494c77a5a0606b6bafecdf7722e62","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"451e15ada92473b46c37ac7d85751c6c542d9156788736edb08510846c807aa5","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"593e3571cb87eb1c990363d1e184cfd091bfa920f1d8233ab9b8bffafc8753f4","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"626800ce28af5178db2305452aa40119998bcbdd63d53a7168853b1c7e730bd3","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"671e3e4773a45e7910b33f7d44f7a3f0d6be8188410cc6c10ced0d14da59716b","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"6d8549ce7ce3d738e38f7dd73b06fae91adfeb0e1534bba97833a20ae84d2f8a","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"7abcaf7cf828a86d797ada0bafb042a8494db58e91375d1805c53de72f0578cb","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"91e2917906e893c4adf51d8e3ad38930cff19ee5cb3009d04823d9b45fe41908","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"9e699244c91afd46d8472b6ccd3aa4e2a0f99db87af440b007dcf8abcbb3ce6d","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"9e87469877ef0917409c14d4d33b60f1b2ccdcdd81485077bc197c0d9d5a5e97","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"a8a2c2adbed4b3b26d51343db3e33c8589838c2cbac6661fcee12fbe4bf879fe","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"aa2abdeccf981853abdf8d84e7171612f5443e4af2a9fb66bb42c41ddd759ab7","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"c8695c56c9968b93a6495331c031bfc4f9defddf58bdeb3d72666b657cc7d8c5","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"d166da666940169c48a559831401ddc71f07bc674b8bac26199e6f32e66bd324","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"db18d58c7751ab0b22cdad498cec1c389a7f4bcbaed367974d93a33204cc2434","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"e19ccc38c497a63fa54dc54b1393fcc47c00e94ab08b2c73966853333434a251","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"f188a0c69f93a586d8064b19fecfad91c62cb72340482fa8197c45527b731e94","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"1081dc1f7ba277c79eab9706dd413878fd827376a9850ff6a2e23d1b663c5155","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"51fb4ad7b246c62ce6ea07724802f78706075364625cc9b0dc59f38a98c159cf","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"52a3e55e5accf233dc0c4dcc31ad7c4bd1abab3ca6c7016627f681951931597f","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"8eb85a382d8ce34de42657ea34ddc180803991e921b8e70810a099fe550ece42","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"b323b711e4f15f03edb6542a780e728151255e7713e3d463bc33d5461bb8ccbf","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"daf2b2ff378776852d48a0372497ce4c540ea79fca95c306e900dac48324c4cd","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"ea6f45f7f8317d27652ca2ba1a9ea637325a720fb6e451e052632539521bf95b","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":26,"runs_affected":26,"failed_runs":26,"run_impact_percent":3.5616438356164384,"anchor_occurrences":26,"after_last_push_count":13,"trend_counts":[0,0,0,0,0,8,7,11],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":26,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":26}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T23:47:51Z\n  Ended:       2026-09-19T00:19:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8441088-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8441088-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T21:03:41Z\n  Ended:       2026-09-18T21:09:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6750464-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6750464-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T17:56:41Z\n  Ended:       2026-09-18T18:02:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9556480-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9556480-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7017/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101084611508441088","occurred_at":"2026-09-18T23:03:02Z","signature_id":"86c45d947e908dd24da56e029e4fb3fb50ede5048c717aadd07439573c0a5e5a","pr_number":7017,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101036649826750464","occurred_at":"2026-09-18T19:52:27Z","signature_id":"f8ce7b6dbb00b186dd3690b8bd0cde21b1999ed29a53641a6363be8c9bc8013e","pr_number":6945,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","signature_id":"1522dde0b9bb5d85e613b7efec8820e9ced25e3c11ff2b7a1c60a5642910649a","pr_number":7010,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6698/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100984946473570304","occurred_at":"2026-09-18T16:10:25Z","signature_id":"f7df40a156b9e56e6a5ddf150c7ac6e35049cd21802df2d33f93003f72173db8","pr_number":6698,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100987718002544640","occurred_at":"2026-09-18T16:01:55Z","signature_id":"db6a9f6935fcb92fd2604ae71dc7832d2001b5cc7b8edb8be0b3faebabe391c0","pr_number":7002,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100937243521716224","occurred_at":"2026-09-18T13:17:26Z","signature_id":"d6f983561c2168d8dc1fb0bb24a4ce58dae5f6ea67b850648a086ef0cc8ced1e","pr_number":7002,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100885599773790208","occurred_at":"2026-09-18T09:52:14Z","signature_id":"4e84895295e21c9ae798ddae187f381094acb88d5bb19172060995a42efdfbcd","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841605895294976","occurred_at":"2026-09-18T06:57:25Z","signature_id":"0d95d8184561f593338e86b1c11b5ad3f6442e5263869a130eac56ca5e35b079","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841559414018048","occurred_at":"2026-09-18T06:57:14Z","signature_id":"b8ef7bc40a5e42cb1d9a25c2c491ed35aa8060746d13168cf09c7ed965a89066","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","signature_id":"49184b986d87df135df2fc3dcc64465876a9918af7c03bd147e1ea6126eccebf","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100779821045911552","occurred_at":"2026-09-18T02:51:54Z","signature_id":"04a95f9ed56a68aa95bb9d5ccdef76442ca4969785b15ebf86b48f429dfe945c","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100680641501728768","occurred_at":"2026-09-17T20:17:48Z","signature_id":"d9ede6c71c00bb7dbee4ddd3fb9aecba02c72aceca980c3acfa9826e45ff50ea","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100563400982532096","occurred_at":"2026-09-17T12:31:48Z","signature_id":"e9c0e585944f16bc753c7dace1676a24a15848f5a821759ef30ea4b61ef466bc","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554684329627648","occurred_at":"2026-09-17T11:56:52Z","signature_id":"8c32a70da3a734350e4252477cd5e4400a822058b8ff76cfef91cedb261d1712","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524951428665344","occurred_at":"2026-09-17T09:59:04Z","signature_id":"a438005342e6e45a592ab0e52281d7a0356c34cf7a20d84dc4af128f220b1b66","pr_number":5724,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524826916556800","occurred_at":"2026-09-17T09:58:39Z","signature_id":"125c7452f650c84b80c4227b5aa637ed38ecce323a860ceae503938b79787746","pr_number":6986,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"795144da2ef4d9ca390a9485406bde85898d43c117d008e0d213f31d6d5042d6","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488384702058496","occurred_at":"2026-09-17T07:33:50Z","signature_id":"77026658b7ddf2ead4aff5831cdfc9e1e01b461fa183028dccacbe6855fbe9a3","pr_number":6952,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100260058020450304","occurred_at":"2026-09-16T16:26:33Z","signature_id":"84adc99394ffdde58d8777dabe660ecf4888bc0533bab685e83dea23a8c7fc1a","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252533153861632","occurred_at":"2026-09-16T15:56:39Z","signature_id":"eb143e51cc4e677a9811a011abde7f9846dfb86171fd5089044d86a86bc3c30e","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"4891bae57c16737882a8247821ec7f6d4e17417de53801a0bcb72589d84d269e","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100228663373991936","occurred_at":"2026-09-16T14:21:48Z","signature_id":"af0482283bab09054590813e51480bea798a5743b7bcc33fa9c63b58d8645bbe","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"6113c0f8fe30ddb0cec1586532a7684c6dc0ec614d444f96448deb0f4ad3a170","pr_number":6879,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132038643290112","occurred_at":"2026-09-16T07:57:51Z","signature_id":"bf9a145d432cea73856e5e5cd0fb2d2856d4a87e3d9fdb2343a2f93ed338bb52","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6923/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100110113200148480","occurred_at":"2026-09-16T06:30:43Z","signature_id":"fac8088074452240e37582cff853c6a9cc2e040277fbf34b005056861933699a","pr_number":6923,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100104517457547264","occurred_at":"2026-09-16T06:08:29Z","signature_id":"5f54bcb248196b637bd98df597f0841d329c7e41ad40465254245b7458522f8c","pr_number":6894,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":16,"runs_affected":16,"failed_runs":16,"run_impact_percent":2.191780821917808,"anchor_occurrences":16,"after_last_push_count":8,"also_in":["INT","PROD","STG"],"trend_counts":[0,0,0,0,0,6,6,4],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":16,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dd1600\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bjbkvcsbld8c\" to finish creating: GET https://rp.j0272768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/641e2016-08be-40cf-9f8e-0e897bdccd1d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/641e2016-08be-40cf-9f8e-0e897bdccd1d\",\n      \"name\": \"641e2016-08be-40cf-9f8e-0e897bdccd1d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T20:46:36.342825036Z\",\n      \"endTime\": \"2026-09-18T21:05:43.553710506Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dd1600\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bjbkvcsbld8c\" to finish creating: GET https://rp.j0272768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/641e2016-08be-40cf-9f8e-0e897bdccd1d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/641e2016-08be-40cf-9f8e-0e897bdccd1d\",\n      \"name\": \"641e2016-08be-40cf-9f8e-0e897bdccd1d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T20:46:36.342825036Z\",\n      \"endTime\": \"2026-09-18T21:05:43.553710506Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.37\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00100a760\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ll6m5x\" in resourcegroup=\"rg-np-version-upgrade-ll6m5x-2g2kcltq9hpv\" to finish creating: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7e813a9-2095-43ef-9c86-073d007f6602\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7e813a9-2095-43ef-9c86-073d007f6602\",\n      \"name\": \"a7e813a9-2095-43ef-9c86-073d007f6602\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T07:07:49.634853409Z\",\n      \"endTime\": \"2026-09-18T07:26:53.294963336Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.37\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00100a760\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ll6m5x\" in resourcegroup=\"rg-np-version-upgrade-ll6m5x-2g2kcltq9hpv\" to finish creating: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7e813a9-2095-43ef-9c86-073d007f6602\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7e813a9-2095-43ef-9c86-073d007f6602\",\n      \"name\": \"a7e813a9-2095-43ef-9c86-073d007f6602\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T07:07:49.634853409Z\",\n      \"endTime\": \"2026-09-18T07:26:53.294963336Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013f8e20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-9ntdl6mdb8xb\" to finish creating: GET https://rp.j8102400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f794a701-689e-4889-b351-a1feb52109b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f794a701-689e-4889-b351-a1feb52109b7\",\n      \"name\": \"f794a701-689e-4889-b351-a1feb52109b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T06:13:08.560808434Z\",\n      \"endTime\": \"2026-09-18T06:32:13.567780508Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013f8e20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-9ntdl6mdb8xb\" to finish creating: GET https://rp.j8102400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f794a701-689e-4889-b351-a1feb52109b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f794a701-689e-4889-b351-a1feb52109b7\",\n      \"name\": \"f794a701-689e-4889-b351-a1feb52109b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T06:13:08.560808434Z\",\n      \"endTime\": \"2026-09-18T06:32:13.567780508Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101033546570272768","occurred_at":"2026-09-18T19:39:52Z","signature_id":"50e9ecb754e15646a314b5b99f973f998a45c92fb4fac3293bbdba2c5f6a2e7e","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","signature_id":"79534399cf979be34c0ffb48ddc762d6841c4a16f58dc222924b468f595a7adb","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6962/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100817340928102400","occurred_at":"2026-09-18T05:20:59Z","signature_id":"fd810f6089fd17b9ae89f5ca382f4365705ce74f6259d695bca5e674d7783e53","pr_number":6962,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6813/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815351901065216","occurred_at":"2026-09-18T05:13:05Z","signature_id":"1b937eb2ed1eae4fc31ae0598f8de6ce2076a500845fc3ea7dcd762870526b2e","pr_number":6813,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100691156575719424","occurred_at":"2026-09-17T20:59:35Z","signature_id":"13adef91a90ab725dab440fb0cd3d153b3bf6f23ccf1898e5f571b9c23787d80","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100678231362375680","occurred_at":"2026-09-17T20:03:18Z","signature_id":"14578415f39945108c1077bc9568d80eb9c57e111ae4ccc29677f2723df82376","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6985/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100532310913847296","occurred_at":"2026-09-17T10:28:23Z","signature_id":"11862ebb0b64914c35ffbbc35bccd2d2fc728333e2922d73613bd21bacfcabf1","pr_number":6985,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100523779246526464","occurred_at":"2026-09-17T09:54:29Z","signature_id":"7fc0697aa9693154afc06e12851d0da5c2c6857a0d678a54183094477e417af7","pr_number":6879,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494069930135552","occurred_at":"2026-09-17T07:56:26Z","signature_id":"41623d8231704f6b3918018ef069742c2b6b4139b736f8af0a971c826a53f099","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100469063225446400","occurred_at":"2026-09-17T06:17:04Z","signature_id":"41037677a8ddb37389a2d414387cf88a833a474c6bb9aecf6b5fe7288cd9a6b8","pr_number":6986,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","signature_id":"11b5fac45f89b3b06fa90df1c77a750f6663c0d038abc00b0bfa209370a40fa8","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100212360202424320","occurred_at":"2026-09-16T13:16:57Z","signature_id":"fca0d524c5ac315fe836fb88fd7ec89c3926bdf9a49356a4f7143c3576597659","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100177539514241024","occurred_at":"2026-09-16T10:58:39Z","signature_id":"4035d63aa7b2b3c2922026477a9808247a9246b7c7d1f67b92c709fe54aaae23","pr_number":6952,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100170313131102208","occurred_at":"2026-09-16T10:29:56Z","signature_id":"f02c430f8d6e71033486f073da10a12fce79eb5455b645129c51f20c2adc1b42","pr_number":6886,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100164075236364288","occurred_at":"2026-09-16T10:05:09Z","signature_id":"05b8123730c99aae90bf4a2eb92268ce4dedb8de1e1191eb555f549c617a26dd","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"b85f0369f91fe32b9c9c2062c2413bb6bf553dd1bd69f6e51a7a27c7ba432aee","pr_number":6791,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":15,"runs_affected":15,"failed_runs":15,"run_impact_percent":2.0547945205479454,"anchor_occurrences":15,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,6,6,3],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":15,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":15}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-19T00:04:30Z\n  Ended:       2026-09-19T00:23:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 6 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-18T18:02:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti77pkq7agiit4e9p66ic3479hnrq5-a7o9q7y9a3g4n2w\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sti77pkq7agiit4e9p66ic3479hnrq5-a7o9q7y9a3g4n2w/router has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-18T18:03:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti77pkq7agiit4e9p66ic3479hnrq5-a7o9q7y9a3g4n2w\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sti77pkq7agiit4e9p66ic3479hnrq5-a7o9q7y9a3g4n2w/router has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-18T18:11:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-18T18:11:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-09-18T18:12:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-09-18T18:12:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T13:28:41Z\n  Ended:       2026-09-18T13:42:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-gdfgt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-18T13:28:41Z\n  Ended:       2026-09-18T13:42:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-gdfgt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/capi-provider has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","signature_id":"61a9d86a254d9ba501b68a3f3cb4e321b30d8b314bb25ec5b72e60af5be20fd0","pr_number":7012,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"035123031fa1d20ace889ddb3bd473206d7c64b2e0e69a0775e5e52e6c40e898","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"b30257b5749c88e7acf4c108618d88d0f18563968e68f249086f8fef88556ee6","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685654236073984","occurred_at":"2026-09-17T20:37:43Z","signature_id":"a25eab3d2aea0534a58060ed60f2bfb99bd3c649f3ff0b27d214e0eebb1f8f68","pr_number":7000,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100523023395196928","occurred_at":"2026-09-17T09:51:23Z","signature_id":"98cf6a34d877442691cd61bb1e77f0e720f13f9f1c1c8cdddc6ad3341afc78ff","pr_number":5724,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"a85ef3a241aa9fee6ef7ff6043d1020492ba3c7fed3de7483ade938f342c7c3e","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100478808179085312","occurred_at":"2026-09-17T06:55:45Z","signature_id":"a995d8a36ccb5cf52cd9d35f8ae8324aa7b3c5eb47ebb28e229463bfb8fb04c6","pr_number":6988,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"6133b883dec1a7ae7199f6897e93bfe8938d4615907efd1f6e96f8f531de45b7","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100385816462233600","occurred_at":"2026-09-17T00:46:16Z","signature_id":"54b42f0c1ae5cc077af2dfa797005520003ed2bd8ca931aabcdf79c00d97c263","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"0d965c3262ee69273c199442b4bed217de7dd1b47c2e6f6c022895186517f6a2","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"314e25f4f6bda2d5748f9fe5aa89f88eaeb725849f7741eef5e0355c2394136c","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"d5ed3091019088eb341f9671128472e74176d896eca057722a5636069c20079f","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","signature_id":"18b82d4d472bbc8192b3a0be8a58b9abe4cf0da273e711ee46164983b7c2c490","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"278a7d810c61d983530553cb6ff8366e5a4d79b77f9c15651d9ebcb3dbeb9e4e","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"89db9db5751f9f51402e00b217747d3b3ea1c62487d699b8eeaa41599ef274e2","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":15,"runs_affected":15,"failed_runs":15,"run_impact_percent":2.0547945205479454,"anchor_occurrences":15,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,10,5,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":15,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":6},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":9}],"full_error_samples":["time=2026-09-17T22:08:23.042Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T22:08:24.010Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-17T22:08:24.819Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=949ef12d6ab429b901225672cf02dc8c2e8f5344e8e4f364f694e8721bc1ed4e portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0316160-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F949ef12d6ab429b901225672cf02dc8c2e8f5344e8e4f364f694e8721bc1ed4e\ntime=2026-09-17T22:38:22.805Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-09-17T21:21:29.249Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T21:21:29.969Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-17T21:21:30.723Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=63b12b492d5d28b0b0b18408fa469c00c31993867db99322c5926f55f6db447a portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6654848-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F63b12b492d5d28b0b0b18408fa469c00c31993867db99322c5926f55f6db447a\ntime=2026-09-17T21:51:28.943Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-09-17T21:19:38.472Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T21:19:39.145Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-17T21:19:39.871Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=f518dca3cc0829098231fe8a127f84b76b966486264714a9e41a0d1fd09997db portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0025600-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff518dca3cc0829098231fe8a127f84b76b966486264714a9e41a0d1fd09997db\ntime=2026-09-17T21:49:38.148Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100701574740316160","occurred_at":"2026-09-17T21:40:59Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100691038216654848","occurred_at":"2026-09-17T20:59:07Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100689397040025600","occurred_at":"2026-09-17T20:52:35Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495912190087168","occurred_at":"2026-09-17T08:03:45Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488996214804480","occurred_at":"2026-09-17T07:36:16Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100332359231475712","occurred_at":"2026-09-16T21:13:51Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100270038647836672","occurred_at":"2026-09-16T17:06:12Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100228886901035008","occurred_at":"2026-09-16T14:22:39Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6779/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100198040831791104","occurred_at":"2026-09-16T12:20:07Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6779,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194885893099520","occurred_at":"2026-09-16T12:07:35Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194111955931136","occurred_at":"2026-09-16T12:04:19Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100192526756483072","occurred_at":"2026-09-16T11:58:12Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100190163022909440","occurred_at":"2026-09-16T11:48:49Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6694/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100169554075324416","occurred_at":"2026-09-16T10:26:55Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6694,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100119215724826624","occurred_at":"2026-09-16T07:06:53Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6972,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":11,"runs_affected":11,"failed_runs":11,"run_impact_percent":1.5068493150684932,"anchor_occurrences":11,"after_last_push_count":8,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,7,1,3],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":11,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b4b300\u003e: \n    failed to create HCP cluster ea-listtxmfvt: failed waiting for cluster=\"ea-listtxmfvt\" in resourcegroup=\"ea-list-b22vvl6sfzgh\" to finish creating: GET https://rp.j8565632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80b95276-553d-4fcb-9838-22d1a7553049\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80b95276-553d-4fcb-9838-22d1a7553049\",\n      \"name\": \"80b95276-553d-4fcb-9838-22d1a7553049\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T23:30:18.658850049Z\",\n      \"endTime\": \"2026-09-18T23:59:19.341659624Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b4b300\u003e: \n    failed to create HCP cluster ea-listtxmfvt: failed waiting for cluster=\"ea-listtxmfvt\" in resourcegroup=\"ea-list-b22vvl6sfzgh\" to finish creating: GET https://rp.j8565632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80b95276-553d-4fcb-9838-22d1a7553049\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80b95276-553d-4fcb-9838-22d1a7553049\",\n      \"name\": \"80b95276-553d-4fcb-9838-22d1a7553049\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T23:30:18.658850049Z\",\n      \"endTime\": \"2026-09-18T23:59:19.341659624Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001084500\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-95vjt5mdwdlk\" to finish creating: GET https://rp.j8771328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ec69745-9888-4c12-ad3c-cfe984e0f2d3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ec69745-9888-4c12-ad3c-cfe984e0f2d3\",\n      \"name\": \"0ec69745-9888-4c12-ad3c-cfe984e0f2d3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T10:07:19.283105586Z\",\n      \"endTime\": \"2026-09-18T10:36:28.169644011Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001084500\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-95vjt5mdwdlk\" to finish creating: GET https://rp.j8771328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ec69745-9888-4c12-ad3c-cfe984e0f2d3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ec69745-9888-4c12-ad3c-cfe984e0f2d3\",\n      \"name\": \"0ec69745-9888-4c12-ad3c-cfe984e0f2d3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T10:07:19.283105586Z\",\n      \"endTime\": \"2026-09-18T10:36:28.169644011Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--xq7n5gbztqbj/cluster-hs-pre-kf6554 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001090320\u003e: \n    failed to create HCP cluster cluster-hs-pre-kf6554: failed waiting for cluster=\"cluster-hs-pre-kf6554\" in resourcegroup=\"rg-hs-pre--xq7n5gbztqbj\" to finish creating: GET https://rp.j2362240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359ebfce-873d-4d6c-8f64-6e12fb114150\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359ebfce-873d-4d6c-8f64-6e12fb114150\",\n      \"name\": \"359ebfce-873d-4d6c-8f64-6e12fb114150\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T04:02:43.525962262Z\",\n      \"endTime\": \"2026-09-18T04:31:43.022568987Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--xq7n5gbztqbj/cluster-hs-pre-kf6554 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001090320\u003e: \n    failed to create HCP cluster cluster-hs-pre-kf6554: failed waiting for cluster=\"cluster-hs-pre-kf6554\" in resourcegroup=\"rg-hs-pre--xq7n5gbztqbj\" to finish creating: GET https://rp.j2362240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359ebfce-873d-4d6c-8f64-6e12fb114150\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359ebfce-873d-4d6c-8f64-6e12fb114150\",\n      \"name\": \"359ebfce-873d-4d6c-8f64-6e12fb114150\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T04:02:43.525962262Z\",\n      \"endTime\": \"2026-09-18T04:31:43.022568987Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","signature_id":"2c332da578489a812e0144aacf458767c886ed8769b126097a13ac0573609822","pr_number":7012,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100877868748771328","occurred_at":"2026-09-18T09:21:30Z","signature_id":"f590a6d9decf3d5d50d5dfcef242c5c8af6ebdbb758d3e90d6fae5136781bae3","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","signature_id":"89409b5febc6093a9cefc9b6ca32afd019b45b211957e3bd23fe77c3eda373f3","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100478808179085312","occurred_at":"2026-09-17T06:55:45Z","signature_id":"ebdc66289f4002396961893fc85f6cbf2d72c402de90efcaab065108518df75a","pr_number":6988,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"7037bf26441097d6041fad63a8ad90e3d7ac2c0bc29160794aafceabed76fa0d","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100293442402783232","occurred_at":"2026-09-16T18:39:12Z","signature_id":"3d2d19c49b473c9b2b5c1bd828e9238e2a0bdd4c4f8d89fe2be19a6ea937a4b7","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","signature_id":"4c86e6598dc5b904362415f87179a0f883f34b4b44e009896f6a7f0de46090e2","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"9ae40ec334e2759457083c312bb3ac6d6fef401546cf0c8751fc4d293a56e796","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","signature_id":"f792cb486e1be9a347881acf7c3d2cd96a120fd226d905b4a6c6fa830ffd406b","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6726/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100185438990897152","occurred_at":"2026-09-16T11:30:02Z","signature_id":"681376edcfb91afa0eef4e45ea02cce5668cf8a6c8cbfde456dd6c90a347a910","pr_number":6726,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132115432607744","occurred_at":"2026-09-16T07:58:09Z","signature_id":"76ed85811ec3a7dce6c999043c0cc9d5ea6c96b382b975abd6e4372f70524c64","pr_number":6964,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":10,"runs_affected":10,"failed_runs":10,"run_impact_percent":1.36986301369863,"anchor_occurrences":10,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,5,1,4],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":10,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":10}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T23:50:30Z\n  Ended:       2026-09-19T00:17:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-18T23:50:30Z\n  Ended:       2026-09-19T00:18:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T13:14:45Z\n  Ended:       2026-09-18T13:44:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-gdfgt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-18T13:14:45Z\n  Ended:       2026-09-18T13:45:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-gdfgt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T10:27:52Z\n  Ended:       2026-09-18T10:54:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8771328-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.161:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8771328/providers/microsoft.monitor/accounts/hcps-j8771328\", namespace=\"ocm-arohcpci01-2stbulpvmbabhuosdje4290vg8cio72d-j5i4m0f4c4k3j8e\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-nsf7x\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2stbulpvmbabhuosdje4290vg8cio72d-j5i4m0f4c4k3j8e/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-18T10:28:56Z\n  Ended:       2026-09-18T10:54:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8771328-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.161:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8771328/providers/microsoft.monitor/accounts/hcps-j8771328\", namespace=\"ocm-arohcpci01-2stbulpvmbabhuosdje4290vg8cio72d-j5i4m0f4c4k3j8e\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-nsf7x\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2stbulpvmbabhuosdje4290vg8cio72d-j5i4m0f4c4k3j8e/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","signature_id":"7dead5f17d20c600ef9a15a67c9915db561b768a43db7c68ada9d07b7b90dfc6","pr_number":7012,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"77db7532989dc335d44e3fd0e347138b7a4aa37045fd816ecfc3761a0789b616","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100877868748771328","occurred_at":"2026-09-18T09:21:30Z","signature_id":"077ee51cbdfb7388e2c8c28578e14f7c26aad49cf417b4ab9a95eda6e094ce1d","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","signature_id":"32caccfe24784ea73efa24f662aed9cbdd2ff753b7f3457b2fae68c38d3d3c3d","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100478808179085312","occurred_at":"2026-09-17T06:55:45Z","signature_id":"78b86d170894f1306383271eb64e5c7231bc9db1d4dade7d27fb2bd571aeb710","pr_number":6988,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","signature_id":"f812f1c6853ffe5ec0bbae33597f7ac343264a944ebc861a726251e8dce290ce","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"b995781a6685881ffdb2f7b53822eec544ca3973eb89f51c67b3781f51c35c18","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","signature_id":"67413ecc4fbc527d42baea24debbddc611e8d90812ae968414dd499232adf3e3","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6726/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100185438990897152","occurred_at":"2026-09-16T11:30:02Z","signature_id":"e361abead0603e0446eac1b70560a55fbc49aa8e7044098f3920029a67178a2a","pr_number":6726,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132115432607744","occurred_at":"2026-09-16T07:58:09Z","signature_id":"2202a1a6bca53c40fa9b99a6151f7edd9b592782635ef9dfab434b81cf067278","pr_number":6964,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":10,"runs_affected":9,"failed_runs":9,"run_impact_percent":1.2328767123287672,"anchor_occurrences":10,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,5,3,2],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":9,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:111]: failed to create node pool \"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001015700\u003e: \n    failed to create NodePool np-zrxbbn: failed waiting for nodepool=\"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\" in resourcegroup=\"rg-hs-pre--gs2tq8nbktl2\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n      \"name\": \"4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T03:55:06.959884593Z\",\n      \"endTime\": \"2026-09-18T03:55:51.664849739Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:111]: failed to create node pool \"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001015700\u003e: \n    failed to create NodePool np-zrxbbn: failed waiting for nodepool=\"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\" in resourcegroup=\"rg-hs-pre--gs2tq8nbktl2\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n      \"name\": \"4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T03:55:06.959884593Z\",\n      \"endTime\": \"2026-09-18T03:55:51.664849739Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017ae060\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-r4g2f9rh44ss\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n      \"name\": \"499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T03:54:50.355767916Z\",\n      \"endTime\": \"2026-09-18T03:55:21.65277157Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017ae060\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-r4g2f9rh44ss\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n      \"name\": \"499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T03:54:50.355767916Z\",\n      \"endTime\": \"2026-09-18T03:55:21.65277157Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001d94018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-g7cwbl5k67j4\" to finish creating: GET https://rp.j8532992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbeee8b5-27cd-473e-a9c2-df2114c56f2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbeee8b5-27cd-473e-a9c2-df2114c56f2b\",\n      \"name\": \"fbeee8b5-27cd-473e-a9c2-df2114c56f2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T00:05:39.3471997Z\",\n      \"endTime\": \"2026-09-18T00:06:19.6272248Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001d94018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-g7cwbl5k67j4\" to finish creating: GET https://rp.j8532992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbeee8b5-27cd-473e-a9c2-df2114c56f2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbeee8b5-27cd-473e-a9c2-df2114c56f2b\",\n      \"name\": \"fbeee8b5-27cd-473e-a9c2-df2114c56f2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T00:05:39.3471997Z\",\n      \"endTime\": \"2026-09-18T00:06:19.6272248Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","signature_id":"6acf378b1d751cabdd2cb29f20e478c64231e5b033214257ad51d0d59a0c2ba7","pr_number":6972,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","signature_id":"f081e78216b7bd4a54e652f3dfd3a2cf8cf19638bb76bda01151759533a92078","pr_number":6972,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721125368532992","occurred_at":"2026-09-17T22:58:38Z","signature_id":"3ad0c7f8513c2d305c0610677386a9460eaa446d3f7dfb59c408d79cec65a7ec","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"0e7b7131df9efbbc641dfa4dfd266b388acf38942df6680d264c1a305036569d","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100420606720741376","occurred_at":"2026-09-17T03:04:31Z","signature_id":"4b99cbe35f0bd3c6197a97e4f313692f0f80c116d5fba8a8d582ebf622976c32","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298527354654720","occurred_at":"2026-09-16T18:59:25Z","signature_id":"bf7ebbd181ec7aa5127a5794e1778bd8570a9bdf75c05ec0454a8c04349e2cae","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100280971763912704","occurred_at":"2026-09-16T17:49:39Z","signature_id":"e4e6c14d747d24bff47997187743a4306c16de10ca6e0660e678007cc3c9c76a","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"37839b6f366af376a2e2dee46f1b956ca9ee9897a768408f797bc256ab2e722b","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"080a9e45d6f4d800d1c8aca157df99a9c02fa085354fe52205b1fcea2ffcae50","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132038643290112","occurred_at":"2026-09-16T07:57:51Z","signature_id":"fb548ce2c81d7ac7aefb5c63ff2cf1623addd485c6b89a7815a4071a2a0cf5c7","pr_number":6503,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":9,"runs_affected":9,"failed_runs":9,"run_impact_percent":1.2328767123287672,"anchor_occurrences":9,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,4,3,2],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":9,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":9}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","signature_id":"e3628da3891e1ec49619dde7547596c92e9e1c1cfe73fca8833c1206090506a8","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488796448493568","occurred_at":"2026-09-17T07:35:28Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"b9b50758a1aa006036c7c1ee58af90cd509c7b1900722ef0229b46ed2f5e8563","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"b9b50758a1aa006036c7c1ee58af90cd509c7b1900722ef0229b46ed2f5e8563","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100196783199096832","occurred_at":"2026-09-16T12:15:07Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"817dbbd2f4e987cf783899d57ba9ab565857a252a91d7741c25fbfc506aff5e8","pr_number":6879,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.095890410958904,"anchor_occurrences":8,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,3,1,4],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":8}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T09:27:59Z\n  Ended:       2026-09-18T09:34:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6818048-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T09:23:28Z\n  Ended:       2026-09-18T09:39:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8030592-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T04:18:48Z\n  Ended:       2026-09-18T04:37:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2362240-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100862764498030592","occurred_at":"2026-09-18T08:21:29Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":6984,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712898639695872","occurred_at":"2026-09-17T22:25:59Z","signature_id":"3b128919de15d094b3050816e84bfdc2f25e06d586dd063b36f0c63378fdbfef","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"660cafcaf8e83347ebad23ce568c8f5b0468c2f68c8566340ecac05676075f1d","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"fe76c2aef36843354f881968133b4c2405287f6a2f2ee3e1e7ab8ab4682897ce","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100153419804184576","occurred_at":"2026-09-16T09:22:48Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":8,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.410958904109589,"anchor_occurrences":8,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,3,2,3],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00156c060\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-nzl72g5md8zn\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n      \"name\": \"a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:31:17.188166352Z\",\n      \"endTime\": \"2026-09-18T17:55:21.133690065Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7a39k8no3d3k6o47iqafudkp0ufb still exists (deletion dispatched at 2026-09-18T17:31:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00156c060\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-nzl72g5md8zn\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n      \"name\": \"a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:31:17.188166352Z\",\n      \"endTime\": \"2026-09-18T17:55:21.133690065Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7a39k8no3d3k6o47iqafudkp0ufb still exists (deletion dispatched at 2026-09-18T17:31:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012e61b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-ghmpzk\" in resourcegroup=\"rg-zstream-upgrade-4-20-nzh4xtf2qkhs\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\",\n      \"name\": \"58c027c6-4639-48fa-ace0-f454c7358b62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:43:42.257649954Z\",\n      \"endTime\": \"2026-09-18T18:07:51.061074345Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7ae2rm414v72bb1h0rpr5prt24in still exists (deletion dispatched at 2026-09-18T17:43:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012e61b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-ghmpzk\" in resourcegroup=\"rg-zstream-upgrade-4-20-nzh4xtf2qkhs\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\",\n      \"name\": \"58c027c6-4639-48fa-ace0-f454c7358b62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:43:42.257649954Z\",\n      \"endTime\": \"2026-09-18T18:07:51.061074345Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7ae2rm414v72bb1h0rpr5prt24in still exists (deletion dispatched at 2026-09-18T17:43:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014723f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listg6frkr\" in resourcegroup=\"ea-list-r7jhwkxx4sdv\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n      \"name\": \"9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:26:18.493088022Z\",\n      \"endTime\": \"2026-09-18T17:50:21.112379819Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7eotueh3r9mn8c08fdvae8djl7o8 still exists (deletion dispatched at 2026-09-18T17:26:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014723f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listg6frkr\" in resourcegroup=\"ea-list-r7jhwkxx4sdv\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n      \"name\": \"9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:26:18.493088022Z\",\n      \"endTime\": \"2026-09-18T17:50:21.112379819Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7eotueh3r9mn8c08fdvae8djl7o8 still exists (deletion dispatched at 2026-09-18T17:26:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"7e8e740680a98bd62be8f0f23d65fc40251d4d5f7742370deded696800b1b588","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"9e65f4af22bf7ec9953560fed8bb49590fa82fc63d7f9f6d24d707a1a503f81c","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"a0f110873c9fc7db9f376a400bf0d8b83aa3eddcd644b62e99f4963c1f15aa39","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"56f9b496d774ec73f62dc8457705f5fe010fad5d913e37425e1aeca77033669a","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"e88abeefa8796da83e3bcaacbe5d5b7d5cc078408bb87028a285969eda618a61","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"697bd9b6305361c1b5307d24f6145b0b596cf49460c3ec8e96a3eae19ea908a8","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"6ba2d7f149e18d77de06d3b79ce4ecb80b3fb20da43bf259cd0570583e9d7af6","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"ff4cffb5ab05b8596815bef0f9113f08c5ccdefbdae02ac3f027a27a7d4486ea","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":0.958904109589041,"anchor_occurrences":7,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,4,2,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":7}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-18T07:08:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j0748928-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-9hl75\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-9hl75 (namespace arobit) on cluster ci01-j0748928-svc has averaged 125.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-18T01:12:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6258560-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-m8n67\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-m8n67 (namespace arobit) on cluster ci01-j6258560-svc has averaged 125.1% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T14:18:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j7954560-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-th462\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-th462 (namespace arobit) on cluster ci01-j7954560-svc has averaged 125.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","signature_id":"476aed8123ee528db23f3663c1332194314ed537bec78310ea649964cd6c166f","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721091256258560","occurred_at":"2026-09-17T22:58:32Z","signature_id":"fffbb7c99a9e3fbfbbfb27296f90fe1eaf381e133c47c2097dc7c5b3440df9c4","pr_number":7000,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","signature_id":"f1e05c2e51f466f034476da1ae61bfe77799ca5b43dd98ad6f2836fda4da614d","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"7cedda9870b8f7ce3e9e32ac805564cf0bac96565ca3328c26875a1d70efde15","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309676229922816","occurred_at":"2026-09-16T19:43:43Z","signature_id":"3a277ae7c258e0424f569baffe0a35ab11aec77aa3384cc607f2298340f9545f","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100285588732121088","occurred_at":"2026-09-16T18:08:00Z","signature_id":"040c54783eeb571a32188be480f0e0617f1911469f8d08b133b147d373df8efb","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100273649465430016","occurred_at":"2026-09-16T17:20:33Z","signature_id":"de52e1f3238a7b10dc025230a0bc30a7b49fe207de3da3e23620f54404272671","pr_number":6894,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":7,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.684931506849315,"anchor_occurrences":7,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,6,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a10e80\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8nfj4dg6xpxk\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/063e2ac2-9588-4f9c-82b9-899a69317856\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/063e2ac2-9588-4f9c-82b9-899a69317856\",\n      \"name\": \"063e2ac2-9588-4f9c-82b9-899a69317856\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:48:07.392021884Z\",\n      \"endTime\": \"2026-09-17T03:17:09.326897061Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a10e80\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8nfj4dg6xpxk\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/063e2ac2-9588-4f9c-82b9-899a69317856\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/063e2ac2-9588-4f9c-82b9-899a69317856\",\n      \"name\": \"063e2ac2-9588-4f9c-82b9-899a69317856\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:48:07.392021884Z\",\n      \"endTime\": \"2026-09-17T03:17:09.326897061Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:90]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eaac80\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-5sd5p2zrd22p\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb287309-affe-4cb6-ba60-0ab97101fffb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb287309-affe-4cb6-ba60-0ab97101fffb\",\n      \"name\": \"cb287309-affe-4cb6-ba60-0ab97101fffb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T21:22:15.559693502Z\",\n      \"endTime\": \"2026-09-16T21:51:23.369287505Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:90]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eaac80\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-5sd5p2zrd22p\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb287309-affe-4cb6-ba60-0ab97101fffb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb287309-affe-4cb6-ba60-0ab97101fffb\",\n      \"name\": \"cb287309-affe-4cb6-ba60-0ab97101fffb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T21:22:15.559693502Z\",\n      \"endTime\": \"2026-09-16T21:51:23.369287505Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:584]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101ca40\u003e: \n    failed to create HCP cluster np-downgrade-ptnt6v: failed waiting for cluster=\"np-downgrade-ptnt6v\" in resourcegroup=\"rg-np-downgrade-ptnt6v-s6586bctkvvs\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b81923cc-2583-43fb-a0ea-baa197fb0d6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b81923cc-2583-43fb-a0ea-baa197fb0d6f\",\n      \"name\": \"b81923cc-2583-43fb-a0ea-baa197fb0d6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:07:45.747660708Z\",\n      \"endTime\": \"2026-09-16T15:36:50.286764481Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:584]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101ca40\u003e: \n    failed to create HCP cluster np-downgrade-ptnt6v: failed waiting for cluster=\"np-downgrade-ptnt6v\" in resourcegroup=\"rg-np-downgrade-ptnt6v-s6586bctkvvs\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b81923cc-2583-43fb-a0ea-baa197fb0d6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b81923cc-2583-43fb-a0ea-baa197fb0d6f\",\n      \"name\": \"b81923cc-2583-43fb-a0ea-baa197fb0d6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:07:45.747660708Z\",\n      \"endTime\": \"2026-09-16T15:36:50.286764481Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"d7f93e547e4be2029c411ab0ce7110ad6d4b2c44c4f85c58ccad50e27c80ca41","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"2ffe72f7718510ccd07fc00e17f58bc0013af474b7348754fe62e7d3c4440904","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"f53d0a32dc007711392e3ac9a6346e4050a79f13a86cf1b09a36647de1c455fb","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"55ad0310438d8f56a4a299f6bab3b8eead74cd3a5f7c47eb326a8464bccf50a4","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"91a400cbca3134720c48834459e3002d3cce5a22f2d380c716d75f65f9d75177","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"b82de203b62ff6756c30d031383d9f2c1cda2b52933916b1090a4b903f7e89b9","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"25f5f479c873e8672b346f86e1757b6d6cdd4c81dcdc1af82909656949ba3709","pr_number":6879,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.821917808219178,"anchor_occurrences":6,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,2,1,3],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":6}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T23:52:29Z\n  Ended:       2026-09-19T00:21:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e1a1529f-88f5-4f8c-8503-a2513cf48144\"\n  Description: Pod ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-18T23:53:28Z\n  Ended:       2026-09-19T00:17:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e1a1529f-88f5-4f8c-8503-a2513cf48144\"\n  Description: Pod ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T13:16:41Z\n  Ended:       2026-09-18T13:44:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"af416a40-e08e-480e-bf5b-da79914398be\"\n  Description: Pod ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-18T13:17:41Z\n  Ended:       2026-09-18T13:45:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"af416a40-e08e-480e-bf5b-da79914398be\"\n  Description: Pod ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T04:31:05Z\n  Ended:       2026-09-18T04:48:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2362240-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2362240/providers/microsoft.monitor/accounts/hcps-j2362240\", namespace=\"ocm-arohcpci01-2st6jp6ln5eij992a4h7v8fe6stngjn4-s9g6g5z2i0s8b2f\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e333cab1-7b1b-43d4-a2a8-c38ce512c9fe\"\n  Description: Pod ocm-arohcpci01-2st6jp6ln5eij992a4h7v8fe6stngjn4-s9g6g5z2i0s8b2f/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","signature_id":"b0cb1249b73e4e17694cee1167aadd3f0fe5951d24186dea9efdb1984ff09dea","pr_number":7012,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"f71f9d0966fc26ad6ff3cd9a2f086d86818c4234b98d4c027fd22b942a6e7ba2","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","signature_id":"b14a5290094a9e0d25292157800f9158e06c44b980a0e58131fba650ea86837b","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100478808179085312","occurred_at":"2026-09-17T06:55:45Z","signature_id":"62ae640dcc38eb25ad05e49560b842c80c55b99017dce7da9cea433518d78675","pr_number":6988,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","signature_id":"18090d26c3a2666e0dfd30f6dc97088df512304fef09dec58c535d88cec6f5bd","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","signature_id":"2c11c083dc96efa9f51e185909eda32654c395f47b2dbb78eaee76c2573a630d","pr_number":6898,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.821917808219178,"anchor_occurrences":6,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,5,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":6}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T02:04:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-6f59c7c94f-hsk8t\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-6f59c7c94f-hsk8t has been in a non-ready state for longer than 5 minutes.","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T21:55:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-dgkdj\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-dgkdj has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T21:55:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-cb282\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-cb282 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T21:55:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-hdnbp\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-hdnbp has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T21:55:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-f58cs\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-f58cs has been in a non-ready state for longer than 5 minutes.","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:45:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-59s8j\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-59s8j has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:45:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-9zlkb\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-9zlkb has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:45:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-t8hdl\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-t8hdl has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:45:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-49tpv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-49tpv has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"9a8fc1c68aeeef65c0fa2d8774b771f9f44e0b8d2622880bad833fdcea513992","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100332359231475712","occurred_at":"2026-09-16T21:13:51Z","signature_id":"4cfee31556334d3f7789166186ab36ba11bfce4b40f49a38dd4a3b6152068d51","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100270038647836672","occurred_at":"2026-09-16T17:06:12Z","signature_id":"ce202b7ce3d72dcf4b71c51db3c021d84810e4ab274c64dae9e86007cff0fec7","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"59897c5f85f7527efbbab27b1d42a8c63b62d2873c4723537141e6e4cb09a74c","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161854826352640","occurred_at":"2026-09-16T09:56:19Z","signature_id":"f46fa77bbf961b7174b5b28e58ea0b344fb9d8b591b31027fca64620b0c955e9","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132066686406656","occurred_at":"2026-09-16T07:57:56Z","signature_id":"5c93d0c045993799007f8590467470292210fd23c8f5530b0425e5382165f354","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"52bfcbbc720001a7392b6a1e08f396579b61443952ed16c2c80c98fe6498c690","failure_pattern":"alert [svc] userJourneyFrontendLatencyP996h30m fired","search_query":"alert [svc] userJourneyFrontendLatencyP996h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP996h30m does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.821917808219178,"anchor_occurrences":6,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,6,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP996h30m does not fire","occurrences":6}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T09:29:21Z\n  Ended:       2026-09-16T09:49:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatencyP996h30m\", cluster=\"ci01-j5155200-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5155200/providers/microsoft.monitor/accounts/services-j5155200\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n  Description: Frontend p99 latency on cluster ci01-j5155200-svc is above 1s on both the 6h and 30m windows (p99 \u003c 1s ARM sync SLO miss).","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T09:46:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatencyP996h30m\", cluster=\"ci01-j3290112-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3290112/providers/microsoft.monitor/accounts/services-j3290112\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n  Description: Frontend p99 latency on cluster ci01-j3290112-svc is above 1s on both the 6h and 30m windows (p99 \u003c 1s ARM sync SLO miss).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T08:53:50Z\n  Ended:       2026-09-16T09:06:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatencyP996h30m\", cluster=\"ci01-j9843328-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9843328/providers/microsoft.monitor/accounts/services-j9843328\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n  Description: Frontend p99 latency on cluster ci01-j9843328-svc is above 1s on both the 6h and 30m windows (p99 \u003c 1s ARM sync SLO miss)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132075855155200","occurred_at":"2026-09-16T07:57:59Z","signature_id":"4e60a01b4827c68ff57b07edc9eab558ce75aad5112478ce2737bee6696f871b","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132038643290112","occurred_at":"2026-09-16T07:57:51Z","signature_id":"9daba40c60f37faf2af38abc3e3474d5917ef206c95bff1353f0bdaf3df9ac3b","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"cbb7931aab2d0ed1916b8d84363dfe1ffc05cf35d9ce851f977710db9cdf561f","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6971/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100127403039789056","occurred_at":"2026-09-16T07:39:25Z","signature_id":"02815353fa0a66ddf20b88ae30c92eac6b08249d84094735fa13cf0c3952861f","pr_number":6971,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6923/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100110113200148480","occurred_at":"2026-09-16T06:30:43Z","signature_id":"d87f770fb784724db359656072889a1f3015ce39f593db987908535455d1ee42","pr_number":6923,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100104517457547264","occurred_at":"2026-09-16T06:08:29Z","signature_id":"d21b60b6e4bcd8f18d5041389daab6c5b24d976173fd091a6cd2e7e5a098f762","pr_number":6894,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.821917808219178,"anchor_occurrences":6,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,2,1,3],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":6}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001a0ed40\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59914-\u003e172.30.0.10:53: read: connection refused\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001a0ed40\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59914-\u003e172.30.0.10:53: read: connection refused\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.002s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000c8c390\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:35668-\u003e172.30.0.10:53: read: connection refused\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.002s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000c8c390\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:35668-\u003e172.30.0.10:53: read: connection refused\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0018c70e0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59154-\u003e172.30.0.10:53: read: connection refused\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0018c70e0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59154-\u003e172.30.0.10:53: read: connection refused\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7003/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100859139801485312","occurred_at":"2026-09-18T08:07:05Z","signature_id":"4553dd78f39dc989ce886fb5e82604c7aa8be953927622430a5564e21a9cc7f0","pr_number":7003,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827581187100672","occurred_at":"2026-09-18T06:01:39Z","signature_id":"25746549fcd7e3d493a2e9010aa4c2f674167bdbce01dd388ff30ebecae482d7","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780868975988736","occurred_at":"2026-09-18T02:56:04Z","signature_id":"3f4f2a1f835d80221855e8810686dfbc430d2c91ebb857edce8074813fea620c","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"41e17c7a4eecb6c1ed4ee23560b530d6b6a42361cc6e3ef0250dbdef28f6d545","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100294017282478080","occurred_at":"2026-09-16T18:41:29Z","signature_id":"30b7eed5c08665fa2cef128c104b6bc116a828b17cff3a0328ac6b1a7aaf5b24","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","signature_id":"e12e1b463aa599590cba976eac3785b82a07ad6fd298aa2e1664ebd292e43daa","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"40f7a558a9f14e3ad84a0dc62150fb9e1dcec198cbfa1c00cbbb104cbc63f3a7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":6,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":6,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,6,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0028a4438\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-j5zkl24x6bwt\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecb9f85-876c-467d-91c9-980ce11c9827\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecb9f85-876c-467d-91c9-980ce11c9827\",\n      \"name\": \"5ecb9f85-876c-467d-91c9-980ce11c9827\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:49:19.070350042Z\",\n      \"endTime\": \"2026-09-16T10:13:22.487477443Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0028a4438\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-j5zkl24x6bwt\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecb9f85-876c-467d-91c9-980ce11c9827\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecb9f85-876c-467d-91c9-980ce11c9827\",\n      \"name\": \"5ecb9f85-876c-467d-91c9-980ce11c9827\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:49:19.070350042Z\",\n      \"endTime\": \"2026-09-16T10:13:22.487477443Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00153e0f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-9dwsgxpprw8c\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\",\n      \"name\": \"9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:44:48.885828252Z\",\n      \"endTime\": \"2026-09-16T10:08:52.4506315Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00153e0f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-9dwsgxpprw8c\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\",\n      \"name\": \"9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:44:48.885828252Z\",\n      \"endTime\": \"2026-09-16T10:08:52.4506315Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000012f78\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-g9bvp2\" in resourcegroup=\"rg-candidate-4-20-pmdbfsk6v6zd\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/834c2939-b42e-48bd-b340-960d8826a429\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/834c2939-b42e-48bd-b340-960d8826a429\",\n      \"name\": \"834c2939-b42e-48bd-b340-960d8826a429\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:49:50.424611242Z\",\n      \"endTime\": \"2026-09-16T10:13:52.492147666Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000012f78\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-g9bvp2\" in resourcegroup=\"rg-candidate-4-20-pmdbfsk6v6zd\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/834c2939-b42e-48bd-b340-960d8826a429\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/834c2939-b42e-48bd-b340-960d8826a429\",\n      \"name\": \"834c2939-b42e-48bd-b340-960d8826a429\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:49:50.424611242Z\",\n      \"endTime\": \"2026-09-16T10:13:52.492147666Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"18c69429807d38cfdd701e251877ce552b6dddaac55af8cb470eb1279a95b526","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"232d76b4c8604eadbe09c019eea03e4b35e8f44b8a88fbcca5afcc985e967ce4","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"9410511803cbf1e3c159d5decef593399c3f8d02958271b7b1f469d232f45e6e","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"9635d65ddbc8d8183553acfbe1c7e3ebf7cfb49f71fb8721193391b355cf2fbd","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"ba6d9f57396d35aba73dd64e7c229f8d7cbb408e58e239c7c2b2065df4e28656","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"d539c69fd361c2a857c348ec75ef524c1710267ecef6f193ceb123bc3e8bce18","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.684931506849315,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,5,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":5}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T22:06:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.104:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1475712/providers/microsoft.monitor/accounts/services-j1475712\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-59mfc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T22:06:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.104:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1475712/providers/microsoft.monitor/accounts/services-j1475712\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-59mfc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:55:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7836672/providers/microsoft.monitor/accounts/services-j7836672\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-zdqrc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:55:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7836672/providers/microsoft.monitor/accounts/services-j7836672\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-zdqrc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T16:45:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T16:45:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T16:46:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T16:46:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100332359231475712","occurred_at":"2026-09-16T21:13:51Z","signature_id":"617d8bea63f7c0a8e5aa65e252453013f2f87eaabc747dea5d193edf0ea92b6a","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100270038647836672","occurred_at":"2026-09-16T17:06:12Z","signature_id":"659a105790bd191ed63ba5bb7e543dad17b2528556030da7e70281aa38365890","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"8cefc20a001466adc6a61bb43c8407f0170786f35f4348a8d83cb4fc19e10669","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161854826352640","occurred_at":"2026-09-16T09:56:19Z","signature_id":"73664eccd132c9d84475a61b44857f75314a6635b79c98d2ae19c6ecf756089a","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132066686406656","occurred_at":"2026-09-16T07:57:56Z","signature_id":"358902433200cc0a70d0c371110e19da053abecdbf25175a02fa1df21c363226","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.684931506849315,"anchor_occurrences":5,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,2,2,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":5}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-19T00:04:52Z\n  Ended:       2026-09-19T00:10:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/services-j8565632\", namespace=\"klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-19T00:04:52Z\n  Ended:       2026-09-19T00:10:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/services-j8565632\", namespace=\"klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 5 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T03:10:30Z\n  Ended:       2026-09-17T03:22:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9076992/providers/microsoft.monitor/accounts/services-j9076992\", namespace=\"klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-6q7r5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T03:10:30Z\n  Ended:       2026-09-17T03:22:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9076992/providers/microsoft.monitor/accounts/services-j9076992\", namespace=\"klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-6q7r5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T03:10:30Z\n  Ended:       2026-09-17T03:22:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9076992/providers/microsoft.monitor/accounts/services-j9076992\", namespace=\"klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-6q7r5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-17T03:11:28Z\n  Ended:       2026-09-17T03:21:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9076992/providers/microsoft.monitor/accounts/services-j9076992\", namespace=\"klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-6q7r5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-17T03:11:28Z\n  Ended:       2026-09-17T03:21:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9076992/providers/microsoft.monitor/accounts/services-j9076992\", namespace=\"klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-6q7r5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","alert fired 16 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T02:29:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T02:30:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T02:35:35Z\n  Ended:       2026-09-17T02:42:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfodlh52rjsn8v0i6tqm6t35c2l0p0\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfodlh52rjsn8v0i6tqm6t35c2l0p0/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-17T02:35:35Z\n  Ended:       2026-09-17T02:42:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfoc87t8ajfhu9n4pqjtbphn862gj1\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfoc87t8ajfhu9n4pqjtbphn862gj1/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-17T02:35:35Z\n  Ended:       2026-09-17T02:42:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfoc8cioja4t99v8qf8fgteg8hp8cq\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfoc8cioja4t99v8qf8fgteg8hp8cq/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-17T02:35:35Z\n  Ended:       2026-09-17T02:42:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfofif68524qhonc25k6st2irl6nvr\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfofif68524qhonc25k6st2irl6nvr/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:42:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfodboatgvtrijvb9itq8g6nv3a0lj\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfodboatgvtrijvb9itq8g6nv3a0lj/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:42:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfod4unnpb3fidghon0kvgohtrm0gi\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfod4unnpb3fidghon0kvgohtrm0gi/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:42:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfod7a9s6uqchgri0b4afl9dc7n19f\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfod7a9s6uqchgri0b4afl9dc7n19f/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:42:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfoda33s4gbilpj50f3r2392mijctf\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfoda33s4gbilpj50f3r2392mijctf/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:43:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfodmtcgprbt25mkk7n1f31gp57v5p\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfodmtcgprbt25mkk7n1f31gp57v5p/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:59:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:42:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfofsi9cr1hrnf33j5m2j2v0anob00\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfofsi9cr1hrnf33j5m2j2v0anob00/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:54:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:42:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfof0kh0mr490ilc2tusdfihbkudsb\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfof0kh0mr490ilc2tusdfihbkudsb/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-09-17T02:37:34Z\n  Ended:       2026-09-17T02:43:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfohaljjdbuho1cfahj1o9d3dj04u4\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfohaljjdbuho1cfahj1o9d3dj04u4/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","signature_id":"a46151e80a37451e13b91d43941493c16950cf32997bf773ab888a8c5db06db6","pr_number":7012,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"69b255fd750c54b7a1baf2eca24627dd8e61f4fdef893acd92470a12832db0f6","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"48e28661b4bfc093fce3e4eccd4ac9c8f3996f35cf8b87e4d8ccb1de279e00a8","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"cd9f479af97dbe998875874266d9ea41c8e58990a5cf6ae60b1ca79fabc876a2","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"3b4c127aae1159c832d5991591f49782774b64887255f529b831eb0acc43aae1","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.684931506849315,"anchor_occurrences":5,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,2,0,3],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.009s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.009s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.000s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.000s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101053926852792320","occurred_at":"2026-09-18T21:00:47Z","signature_id":"453c681e05619434e546781762e4d8f81384d3fa3b09f4cc9d65ac2b7dc22b1d","pr_number":6639,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827581187100672","occurred_at":"2026-09-18T06:01:39Z","signature_id":"ee111baab05f6b8ee26f353e865437e1bffda7140d9fc154398f141cd776e6f7","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6813/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815351901065216","occurred_at":"2026-09-18T05:13:05Z","signature_id":"453c681e05619434e546781762e4d8f81384d3fa3b09f4cc9d65ac2b7dc22b1d","pr_number":6813,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","signature_id":"ee111baab05f6b8ee26f353e865437e1bffda7140d9fc154398f141cd776e6f7","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100164075236364288","occurred_at":"2026-09-16T10:05:09Z","signature_id":"186ac175cc37e33f1eaa9e81000b9ad2fcf1670566cc6849f36ba3c782d224b2","pr_number":6688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.684931506849315,"anchor_occurrences":5,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,3,0,2],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001bed8b0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001bed8b0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.008s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000909420\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.008s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000909420\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.000s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0007c5690\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.000s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0007c5690\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101053926852792320","occurred_at":"2026-09-18T21:00:47Z","signature_id":"6566438295d27151c619c08c959579eb7709fb58c49e3c34c87dd1ec1d1e808b","pr_number":6639,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","signature_id":"7193ae4d62017e53f8c9422389a5405738a2e8a036e8555e76291f330d89920e","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298495381475328","occurred_at":"2026-09-16T18:59:17Z","signature_id":"6b84028a0066d5af15d34aa2f3390d5b3b8c5439b097d46537c60eee00e895c1","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100243112008880128","occurred_at":"2026-09-16T15:19:04Z","signature_id":"e0666d1ee60ed5c6d4f663533ea9b4f979d5b8183e061f975c9361ca0cafa025","pr_number":6923,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100237146764873728","occurred_at":"2026-09-16T14:55:30Z","signature_id":"493948c9ce416bdc516799e228ff28ed99f8807ce0868ea1c0984eabe1af5cd2","pr_number":6752,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b414f524aec7c4ea3c6d464450b9af6f734740715a61aff1c3a8f53547f62641","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":5,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.547945205479452,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,3,2,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000648280\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qghsblcj4pgk\" to finish creating: GET https://rp.j4009472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\",\n      \"name\": \"f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T23:29:11.075687263Z\",\n      \"endTime\": \"2026-09-17T23:58:14.591633856Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000648280\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qghsblcj4pgk\" to finish creating: GET https://rp.j4009472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\",\n      \"name\": \"f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T23:29:11.075687263Z\",\n      \"endTime\": \"2026-09-17T23:58:14.591633856Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015ec480\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-4llzpkxqdt5h\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/98da8819-6ed6-431a-b25f-78eab8ef8dd9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/98da8819-6ed6-431a-b25f-78eab8ef8dd9\",\n      \"name\": \"98da8819-6ed6-431a-b25f-78eab8ef8dd9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:12:29.574806708Z\",\n      \"endTime\": \"2026-09-17T13:41:32.177488403Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015ec480\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-4llzpkxqdt5h\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/98da8819-6ed6-431a-b25f-78eab8ef8dd9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/98da8819-6ed6-431a-b25f-78eab8ef8dd9\",\n      \"name\": \"98da8819-6ed6-431a-b25f-78eab8ef8dd9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:12:29.574806708Z\",\n      \"endTime\": \"2026-09-17T13:41:32.177488403Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e4c880\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7vfjq4g5mpkz\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\",\n      \"name\": \"fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:25:02.43485078Z\",\n      \"endTime\": \"2026-09-16T15:54:04.537444594Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e4c880\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7vfjq4g5mpkz\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\",\n      \"name\": \"fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:25:02.43485078Z\",\n      \"endTime\": \"2026-09-16T15:54:04.537444594Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"3952b4207ca326cafd047aae37e1541b9c1921b3b3a1de670914b6430814079b","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"d7b3a6738a3142cf28e35b425618e47c2d78a5ec741df6019ce6ae2b560d9004","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"5f788a7a822406c498eab147df2fe3fad0c3f67d39005f710714ed9d67ac601d","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"976862ec83bbad292378adfc6b6876d3447587a76e0f813a8827a52bd7db4bbc","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"4c23de01cdd9b886d00d4c9a0959f33955c73d1a556a45314d353da1da388706","pr_number":6976,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"575b0de6701cb1cdcdd3b5c4e6523248471d3aa67c3c0c1fed243233e4014b33","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Monitor","search_query":"ERROR CODE: ResourceGroupNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"AMA Metrics service metrics from SVC and MGMT clusters should be present in Azure Monitor","occurrences":5,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.410958904109589,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,4,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"AMA Metrics service metrics from SVC and MGMT clusters should be present in Azure Monitor","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-j4886144 in resource group hcp-underlay-ci01-j4886144\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b90a00\u003e: \n    failed to get workspace services-j4886144: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4886144/providers/Microsoft.Monitor/accounts/services-j4886144\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j4886144' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-j4886144 in resource group hcp-underlay-ci01-j4886144\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b90a00\u003e: \n    failed to get workspace services-j4886144: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4886144/providers/Microsoft.Monitor/accounts/services-j4886144\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j4886144' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:74]: failed to look up HCP Prometheus endpoint\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00072e820\u003e: \n    failed to get workspace hcps-j3128064: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3128064/providers/Microsoft.Monitor/accounts/hcps-j3128064\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j3128064' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:74]: failed to look up HCP Prometheus endpoint\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00072e820\u003e: \n    failed to get workspace hcps-j3128064: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3128064/providers/Microsoft.Monitor/accounts/hcps-j3128064\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j3128064' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-j3128064 in resource group hcp-underlay-ci01-j3128064\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010489e0\u003e: \n    failed to get workspace services-j3128064: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3128064/providers/Microsoft.Monitor/accounts/services-j3128064\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j3128064' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-j3128064 in resource group hcp-underlay-ci01-j3128064\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010489e0\u003e: \n    failed to get workspace services-j3128064: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3128064/providers/Microsoft.Monitor/accounts/services-j3128064\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j3128064' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"1e24a52fcfa8fdc207f5da162e136847f4db22f73c90f28694d25d387bc9480d","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","signature_id":"1d95e250094820742e5d106c9a6198395eff4115c52cfbf3176662d0996cbb8f","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","signature_id":"2300bcf0e83155e35375beb6ab2cd2dd1e60df39d954749c7fd210ed41e13cde","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"4a840acfa7ba40488e61bba537bd235d758cbe0b71e71c1cdf5af1d8876188b2","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"e60b8a3f0f846e4caf38925534828b81ff9b59a90b098e189f25e9987eb8d737","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fe6b01560ac49646fd0a46c48f672cf2fc0ebbf844ea4576bb0cab7a46df7ce8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":5,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.273972602739726,"anchor_occurrences":5,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,4,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f90a40\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8x87dzwxvb96\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3edfae6e-fb2d-4e52-a859-1500f9def9f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3edfae6e-fb2d-4e52-a859-1500f9def9f6\",\n      \"name\": \"3edfae6e-fb2d-4e52-a859-1500f9def9f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:04:32.691734276Z\",\n      \"endTime\": \"2026-09-17T02:33:33.968010236Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f90a40\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8x87dzwxvb96\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3edfae6e-fb2d-4e52-a859-1500f9def9f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3edfae6e-fb2d-4e52-a859-1500f9def9f6\",\n      \"name\": \"3edfae6e-fb2d-4e52-a859-1500f9def9f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:04:32.691734276Z\",\n      \"endTime\": \"2026-09-17T02:33:33.968010236Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:103]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001f3c120\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sxmn5zhcsvjn\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a019abb2-ae16-4218-9707-6f7dc73a3eb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a019abb2-ae16-4218-9707-6f7dc73a3eb3\",\n      \"name\": \"a019abb2-ae16-4218-9707-6f7dc73a3eb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:08:36.527287031Z\",\n      \"endTime\": \"2026-09-16T15:37:40.20476418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:103]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001f3c120\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sxmn5zhcsvjn\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a019abb2-ae16-4218-9707-6f7dc73a3eb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a019abb2-ae16-4218-9707-6f7dc73a3eb3\",\n      \"name\": \"a019abb2-ae16-4218-9707-6f7dc73a3eb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:08:36.527287031Z\",\n      \"endTime\": \"2026-09-16T15:37:40.20476418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--7tj5b6r9d28k/cluster-hs-pre-rwztb2 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ee48a0\u003e: \n    failed to create HCP cluster cluster-hs-pre-rwztb2: failed waiting for cluster=\"cluster-hs-pre-rwztb2\" in resourcegroup=\"rg-hs-pre--7tj5b6r9d28k\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0015f221-0420-4231-b96b-76ce90032f9d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0015f221-0420-4231-b96b-76ce90032f9d\",\n      \"name\": \"0015f221-0420-4231-b96b-76ce90032f9d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:07:38.092292569Z\",\n      \"endTime\": \"2026-09-16T15:36:40.334174064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--7tj5b6r9d28k/cluster-hs-pre-rwztb2 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ee48a0\u003e: \n    failed to create HCP cluster cluster-hs-pre-rwztb2: failed waiting for cluster=\"cluster-hs-pre-rwztb2\" in resourcegroup=\"rg-hs-pre--7tj5b6r9d28k\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0015f221-0420-4231-b96b-76ce90032f9d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0015f221-0420-4231-b96b-76ce90032f9d\",\n      \"name\": \"0015f221-0420-4231-b96b-76ce90032f9d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:07:38.092292569Z\",\n      \"endTime\": \"2026-09-16T15:36:40.334174064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"9020e6f833dc38c47ccef3d4182afa72cebc722f85d46ac12ebd56299acce896","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"0bf39d4aab091ac51c471f05a7fa67a4d591d3eb93555a71c26170936f523c5f","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"a64a06ed18c7d9e35b003b4339972fe04157c007a8cd68d29875724bb6a184d4","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"b2b0b3c510ba3d2ec5bcda27a848edbf0f6e4130232ba677304593697d08e0b2","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"e41f400f85ccdcde13c1223820f617791e8453915fe2b4d9d6f910f58c9aac3e","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.547945205479452,"anchor_occurrences":4,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,2,1,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:183]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:183]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410","failure_pattern":"ERROR CODE: DeploymentStackDeploymentFailed","search_query":"ERROR CODE: DeploymentStackDeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.547945205479452,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,2,0,2],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","occurrences":4}],"full_error_samples":["time=2026-09-18T21:50:35.372Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n  Kind: ARMStack\\n  Template: templates/monitoring.bicep\\n  Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-09-18T21:51:25.579Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/04392d3b-593d-406d-ae87-240d0d6c1c70\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/04392d3b-593d-406d-ae87-240d0d6c1c70\\\",\\n  \\\"name\\\": \\\"04392d3b-593d-406d-ae87-240d0d6c1c70\\\",\\n  \\\"status\\\": \\\"failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n    \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'aad2e336-6c43-473c-997c-4a60a482583a'.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4937088/providers/Microsoft.Resources/deployments/y5flko475afn22dntfyewzfj7gfkbiwdmj7ezjquufwse-26091821etad3\\\",\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n        \\\"details\\\": [\\n          {\\n            \\\"code\\\": \\\"BadRequest\\\",\\n            \\\"message\\\": \\\"Property name: Rules[0].Expression, Attempted value: null, Error: 1:237: parse error: unknown escape sequence U+002E '.' Activity ID: b8850252-9aac-448f-9581-e29b2b8a10a8.\\\"\\n          }\\n        ]\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-18T15:40:00.669Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n  Kind: ARMStack\\n  Template: templates/monitoring.bicep\\n  Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-09-18T15:42:51.615Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/eastus2/deploymentStackOperationStatus/326eddf6-9965-4299-86de-7e1eb754418c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/eastus2/deploymentStackOperationStatus/326eddf6-9965-4299-86de-7e1eb754418c\\\",\\n  \\\"name\\\": \\\"326eddf6-9965-4299-86de-7e1eb754418c\\\",\\n  \\\"status\\\": \\\"failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n    \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'e78fa208-8c43-45b6-a80c-57ad93c6a044'.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4742016/providers/Microsoft.Resources/deployments/ka7tdccaiavlrtibmy3y5s7tfhcs6gpzxc6p4xe5pchjy-26091815bptz8\\\",\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n        \\\"details\\\": [\\n          {\\n            \\\"code\\\": \\\"BadRequest\\\",\\n            \\\"message\\\": \\\"Property name: Rules[0].Expression, Attempted value: null, Error: 1:237: parse error: unknown escape sequence U+002E '.' Activity ID: d9b78f81-a4e0-4b15-9e4d-75e6f5c342e3.\\\"\\n          }\\n        ]\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-16T18:22:02.494Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n  Kind: ARMStack\\n  Template: templates/monitoring.bicep\\n  Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-09-16T18:22:51.675Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/canadacentral/deploymentStackOperationStatus/1df7c411-d9ac-443d-ac5d-b95681f8419b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/canadacentral/deploymentStackOperationStatus/1df7c411-d9ac-443d-ac5d-b95681f8419b\\\",\\n  \\\"name\\\": \\\"1df7c411-d9ac-443d-ac5d-b95681f8419b\\\",\\n  \\\"status\\\": \\\"failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n    \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'd478c792-be07-45be-a743-97b4e94abf15'.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5982976/providers/Microsoft.Resources/deployments/nbyngfosfawbsx5vtxb2uo6lmfim2rbaopp3emm2zwlmm-260916186gla8\\\",\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n        \\\"details\\\": [\\n          {\\n            \\\"code\\\": \\\"BadRequest\\\",\\n            \\\"message\\\": \\\"Property name: Rules[0].Expression, Attempted value: null, Error: 1:237: parse error: unknown escape sequence U+002E '.' Activity ID: 95e8cbf4-5734-47ec-859e-d6ed81e0056f.\\\"\\n          }\\n        ]\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101062836254937088","occurred_at":"2026-09-18T21:36:30Z","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","pr_number":6965,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100966207174742016","occurred_at":"2026-09-18T15:08:20Z","signature_id":"e4c3fb1587b09488a2265155d7a6787ca94e35b729bce621d58ebb8765ea7ef1","pr_number":6965,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100283069305982976","occurred_at":"2026-09-16T17:57:59Z","signature_id":"2a5637582a78b2337c484f5a7a9ff7f9d84aa702c7b086e89182f1549eb13d57","pr_number":6965,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100265336732389376","occurred_at":"2026-09-16T16:47:31Z","signature_id":"2a5637582a78b2337c484f5a7a9ff7f9d84aa702c7b086e89182f1549eb13d57","pr_number":6965,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.547945205479452,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,4,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":4}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T22:05:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j1475712-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j1475712-svc has been unreachable for 15 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:55:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j7836672-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j7836672-svc has been unreachable for 15 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T10:51:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j6352640-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j6352640-svc has been unreachable for 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100332359231475712","occurred_at":"2026-09-16T21:13:51Z","signature_id":"9b1a4c0162aa96d1ba2cfc1a58fe8450105df2f95a84e56b0c0206685fdb86fb","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100270038647836672","occurred_at":"2026-09-16T17:06:12Z","signature_id":"c1988f523c9b1951a937f22e869077937abcdb055a057f4fd9deb538ebdeeab8","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161854826352640","occurred_at":"2026-09-16T09:56:19Z","signature_id":"40d8f345b61cc5a9851d9600ef3889f30540178c4979cc679d48f241649fe011","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132066686406656","occurred_at":"2026-09-16T07:57:56Z","signature_id":"cdf1b6098bdd9c6cf95b2228b3d492b2e30cb0b5e195c0384f719de68055f224","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c71ebc2a93d9fc10a75dcaf39a2b1325de6379c3891238b64d1e4e8f4281df9a","failure_pattern":"failed to get kusto cluster \u003ckusto-cluster\u003e from config: configuration: key kusto not found","search_query":"failed to get kusto cluster name from config: configuration: key kusto not found","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.547945205479452,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,2,2,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000aac060\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000aac060\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0018b8060\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0018b8060\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0012e60a0\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0012e60a0\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100556682735128576","occurred_at":"2026-09-17T12:05:14Z","signature_id":"99c5a5b19f403073edd6e8406f2272a3fda7ff44e9a289f5c999503a39917368","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"daa3fa7c154560025dfba9dba52f2111304f2d047aafee6e951b7d85e52a0fdc","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","signature_id":"9b7d539624448efd4996e6b535e22c740df41bfa52b50eab26733ce795540f15","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"12729704797975e796e396a1c7d5eb085f228e272a9762c5cf68851df1b74ece","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.410958904109589,"anchor_occurrences":4,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,4,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a9c270\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-fqmsfk\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-lqr7gtcm4srq\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\",\n      \"name\": \"3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:13:26.770029811Z\",\n      \"endTime\": \"2026-09-16T17:37:32.768384818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j3fnpr9m9cqfjaohaiq73k6sjvpi still exists (deletion dispatched at 2026-09-16T17:13:27Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a9c270\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-fqmsfk\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-lqr7gtcm4srq\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\",\n      \"name\": \"3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:13:26.770029811Z\",\n      \"endTime\": \"2026-09-16T17:37:32.768384818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j3fnpr9m9cqfjaohaiq73k6sjvpi still exists (deletion dispatched at 2026-09-16T17:13:27Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014240f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xbmtbzvlpclx\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d85e799c-786e-4172-95d1-86693923282f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d85e799c-786e-4172-95d1-86693923282f\",\n      \"name\": \"d85e799c-786e-4172-95d1-86693923282f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:32:57.751739191Z\",\n      \"endTime\": \"2026-09-16T17:57:03.956555792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j51n0bvbqmvanc2bru5bjg1kuddr still exists (deletion dispatched at 2026-09-16T17:32:58Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014240f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xbmtbzvlpclx\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d85e799c-786e-4172-95d1-86693923282f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d85e799c-786e-4172-95d1-86693923282f\",\n      \"name\": \"d85e799c-786e-4172-95d1-86693923282f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:32:57.751739191Z\",\n      \"endTime\": \"2026-09-16T17:57:03.956555792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j51n0bvbqmvanc2bru5bjg1kuddr still exists (deletion dispatched at 2026-09-16T17:32:58Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000012e40\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-wbh9jqvv44p7\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9db0308e-c5ac-4693-b367-d76dc77af863\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9db0308e-c5ac-4693-b367-d76dc77af863\",\n      \"name\": \"9db0308e-c5ac-4693-b367-d76dc77af863\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:29:44.443897591Z\",\n      \"endTime\": \"2026-09-16T15:53:44.471117038Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss64i7i766p1sol3r0dalpub5p7l53h still exists (deletion dispatched at 2026-09-16T15:29:46Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000012e40\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-wbh9jqvv44p7\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9db0308e-c5ac-4693-b367-d76dc77af863\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9db0308e-c5ac-4693-b367-d76dc77af863\",\n      \"name\": \"9db0308e-c5ac-4693-b367-d76dc77af863\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:29:44.443897591Z\",\n      \"endTime\": \"2026-09-16T15:53:44.471117038Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss64i7i766p1sol3r0dalpub5p7l53h still exists (deletion dispatched at 2026-09-16T15:29:46Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"4b15da3ce1591e33d2696652650de2d0cdc6c841165c33a2a923881126078487","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"4f20aa230f0afb2f03a0196f22a9def6c79b679a7bc9bc79347900fd06c241a8","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"b79349f0a2b72f52e04da9ac4e8ef7179dd24862a80796349270662d77641269","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222891458039808","occurred_at":"2026-09-16T13:58:52Z","signature_id":"8491d54e1e6f9d99de4d3a50c4e2d20f3eb57611f24ff8bb3d09a72d4b3c8424","pr_number":6952,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a6bab59ff05e72a771d612966dd1e7024c1a3d21427661ca54c8003eb2c7e563","failure_pattern":"ERROR CODE: DeploymentFailed; detail code GetMSICredentialError; detail message Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.410958904109589,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,0,2],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1}],"full_error_samples":["time=2026-09-18T03:37:37.864Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-18T03:37:38.393Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-18T03:37:39.053Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=9d03ef43fb389999fc408f90bd02e6c94ce2721f8e17a8400c8a777195fbc298 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1401344-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9d03ef43fb389999fc408f90bd02e6c94ce2721f8e17a8400c8a777195fbc298\ntime=2026-09-18T03:40:39.648Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1401344-mgmt-1/providers/Microsoft.Resources/deployments/9d03ef43fb389999fc408f90bd02e6c94ce2721f8e17a8400c8a777195fbc298/operationStatuses/08584119042267536349\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1401344-mgmt-1/providers/Microsoft.Resources/deployments/cluster-sr7huchijtjyg\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1401344-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j1401344-mgmt-1\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"GetMSICredentialError\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-18T01:22:54.255Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-18T01:22:54.877Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-18T01:22:55.881Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=20d345ced78ed93f1fcc83ca672f15621e19f0c6572d4980fecd14d77cfa1a28 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9648512-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F20d345ced78ed93f1fcc83ca672f15621e19f0c6572d4980fecd14d77cfa1a28\ntime=2026-09-18T01:24:56.385Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9648512-mgmt-2/providers/Microsoft.Resources/deployments/20d345ced78ed93f1fcc83ca672f15621e19f0c6572d4980fecd14d77cfa1a28/operationStatuses/08584119123100595444\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9648512-mgmt-2/providers/Microsoft.Resources/deployments/cluster-zhmgezzmxfkbc\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9648512-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j9648512-mgmt-2\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"GetMSICredentialError\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-16T12:27:55.094Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-09-16T12:27:56.479Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-09-16T12:27:57.833Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=b489aeac3f7b0efc7498443b42a85f2f65ee62ab4f82355239f486d3dbc78483 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5020544-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fb489aeac3f7b0efc7498443b42a85f2f65ee62ab4f82355239f486d3dbc78483\ntime=2026-09-16T12:31:28.399Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5020544-svc/providers/Microsoft.Resources/deployments/b489aeac3f7b0efc7498443b42a85f2f65ee62ab4f82355239f486d3dbc78483/operationStatuses/08584120452084424414\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5020544-svc/providers/Microsoft.Resources/deployments/cluster-6sk23df5akobi\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5020544-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j5020544-svc\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"GetMSICredentialError\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6984/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784773881401344","occurred_at":"2026-09-18T03:11:35Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6984,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100751960339648512","occurred_at":"2026-09-18T01:00:47Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100192568775020544","occurred_at":"2026-09-16T11:58:22Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6964,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.410958904109589,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T22:08:21Z\n  Ended:       2026-09-18T22:18:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j4790528-svc\", component=\"frontend\", method=\"delete\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j4790528-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T18:04:41Z\n  Ended:       2026-09-18T18:24:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j9556480-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j9556480-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T13:28:52Z\n  Ended:       2026-09-18T13:38:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j6576768-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n  Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j6576768-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7018/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101049380864790528","occurred_at":"2026-09-18T20:43:02Z","signature_id":"900336073fa1807e3fa5eb270059d6e459df8256995e512e7d9ae978fff82f8c","pr_number":7018,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","signature_id":"f6e1d34656f1ce37365020fae4967ac3b4ae64788fb876cb79e6b050b731b2fe","pr_number":7010,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100917027756576768","occurred_at":"2026-09-18T11:57:07Z","signature_id":"85fef6c172f8238e32065ce9be94996092fd65499dbb48afe109749b4511663e","pr_number":6939,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gn4q6q4pqf7b\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.410958904109589,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,1,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0031a8160\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-p9htbjsmj226\" to finish creating: GET https://rp.j6818048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\",\n      \"name\": \"1616a745-fc49-408c-a84f-41679708fa49\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T09:27:23.7095894Z\",\n      \"endTime\": \"2026-09-18T09:28:03.46142651Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0031a8160\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-p9htbjsmj226\" to finish creating: GET https://rp.j6818048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\",\n      \"name\": \"1616a745-fc49-408c-a84f-41679708fa49\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T09:27:23.7095894Z\",\n      \"endTime\": \"2026-09-18T09:28:03.46142651Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0009e8ec0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qbcpd5p47nxj\" to finish creating: GET https://rp.j4979456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3bad1b6-65e5-4da7-a96f-1c7006af3c80\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3bad1b6-65e5-4da7-a96f-1c7006af3c80\",\n      \"name\": \"f3bad1b6-65e5-4da7-a96f-1c7006af3c80\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:12:19.756903274Z\",\n      \"endTime\": \"2026-09-17T13:12:54.770011764Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0009e8ec0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qbcpd5p47nxj\" to finish creating: GET https://rp.j4979456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3bad1b6-65e5-4da7-a96f-1c7006af3c80\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3bad1b6-65e5-4da7-a96f-1c7006af3c80\",\n      \"name\": \"f3bad1b6-65e5-4da7-a96f-1c7006af3c80\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:12:19.756903274Z\",\n      \"endTime\": \"2026-09-17T13:12:54.770011764Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0011eef40\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gn4q6q4pqf7b\" to finish creating: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c302cb08-5816-4f3c-a786-daa45a7d6406\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c302cb08-5816-4f3c-a786-daa45a7d6406\",\n      \"name\": \"c302cb08-5816-4f3c-a786-daa45a7d6406\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:12:22.418838081Z\",\n      \"endTime\": \"2026-09-16T15:13:01.450563178Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0011eef40\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gn4q6q4pqf7b\" to finish creating: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c302cb08-5816-4f3c-a786-daa45a7d6406\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c302cb08-5816-4f3c-a786-daa45a7d6406\",\n      \"name\": \"c302cb08-5816-4f3c-a786-daa45a7d6406\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:12:22.418838081Z\",\n      \"endTime\": \"2026-09-16T15:13:01.450563178Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","signature_id":"1ff84f075d12052118fcca1c651ed549b6c5f0208069e0615f46732cae26b19e","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554708664979456","occurred_at":"2026-09-17T11:57:12Z","signature_id":"7dda8cc2984aa1b8ad7daf00d042d0064ecf3188c6f0d908489ee4571d215e40","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"162026bfd655fc4db32e7049c532e8d81f8bd34a8e4e6625145cd5a31c7568a3","pr_number":6976,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"50012cc84060fc59db5e840f9933bf1a542baf6598061d9d1e54da53aa955055","failure_pattern":"permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews","search_query":"permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.410958904109589,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,2,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.086s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001018980\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/volumeattachments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiextensions.k8s.io/customresourcedefinitions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list policy/poddisruptionbudgets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list autoscaling/horizontalpodautoscalers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list coordination.k8s.io/leases: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list discovery.k8s.io/endpointslices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list certificates.k8s.io/certificatesigningrequests: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list scheduling.k8s.io/priorityclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list node.k8s.io/runtimeclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/mutatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/validatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiregistration.k8s.io/apiservices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/clusterversions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/clusteroperators: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/infrastructures: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list security.openshift.io/securitycontextconstraints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list route.openshift.io/routes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list image.openshift.io/imagestreams: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.086s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001018980\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/volumeattachments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiextensions.k8s.io/customresourcedefinitions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list policy/poddisruptionbudgets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list autoscaling/horizontalpodautoscalers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list coordination.k8s.io/leases: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list discovery.k8s.io/endpointslices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list certificates.k8s.io/certificatesigningrequests: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list scheduling.k8s.io/priorityclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list node.k8s.io/runtimeclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/mutatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/validatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiregistration.k8s.io/apiservices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/clusterversions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/clusteroperators: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/infrastructures: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list security.openshift.io/securitycontextconstraints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list route.openshift.io/routes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list image.openshift.io/imagestreams: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.100s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001341f60\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.100s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001341f60\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.093s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000ff3040\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/volumeattachments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiextensions.k8s.io/customresourcedefinitions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list policy/poddisruptionbudgets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list autoscaling/horizontalpodautoscalers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list coordination.k8s.io/leases: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list discovery.k8s.io/endpointslices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list certificates.k8s.io/certificatesigningrequests: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list scheduling.k8s.io/priorityclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list node.k8s.io/runtimeclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/mutatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/validatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.093s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000ff3040\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/volumeattachments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiextensions.k8s.io/customresourcedefinitions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list policy/poddisruptionbudgets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list autoscaling/horizontalpodautoscalers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list coordination.k8s.io/leases: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list discovery.k8s.io/endpointslices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list certificates.k8s.io/certificatesigningrequests: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list scheduling.k8s.io/priorityclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list node.k8s.io/runtimeclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/mutatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/validatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495749199433728","occurred_at":"2026-09-17T08:03:06Z","signature_id":"d145278289b57df8d05b7f53866f25f2a3e6803008a36d7c5b0f96179504086e","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100285588732121088","occurred_at":"2026-09-16T18:08:00Z","signature_id":"7f3c6a5d4502814ef937d1139cc649fca2dcf5ac7887cdb4d24e35c02e2550c3","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100188464501755904","occurred_at":"2026-09-16T11:42:04Z","signature_id":"b62e9e3d53137e1d7201b9bf6b9b273592900eb1294c925f09b1fe4ee1b51c90","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wngrxcmwz6zh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-wngrxcmwz6zh, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.410958904109589,"anchor_occurrences":3,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,1,2,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f1dd40\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-xrnlcrchtggh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-xrnlcrchtggh, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f1dd40\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-xrnlcrchtggh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-xrnlcrchtggh, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014fc8a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-zvg8dvqqf6zz\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zvg8dvqqf6zz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014fc8a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-zvg8dvqqf6zz\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zvg8dvqqf6zz, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009db7a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wngrxcmwz6zh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-wngrxcmwz6zh, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009db7a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wngrxcmwz6zh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-wngrxcmwz6zh, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495749199433728","occurred_at":"2026-09-17T08:03:06Z","signature_id":"d7f2fd6c375699dace4737f150cc1d0c82379eed84d0d265ff57b17f49ba73d8","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"a4ab9812dd60d9325a7127735e40ad640073cecf361f77bd3c75a60c7b17bd98","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"f366c8a9d3b487352e0d087e494e88beecbef611270fb1a6b15974fec8f76394","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.273972602739726,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017e8150\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-5mwmp59kkrll\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\",\n      \"name\": \"0000780c-679b-479e-b61c-e08211991339\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:27:09.508692971Z\",\n      \"endTime\": \"2026-09-18T17:51:11.117212687Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti77pkq7agiit4e9p66ic3479hnrq5 still exists (deletion dispatched at 2026-09-18T17:27:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017e8150\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-5mwmp59kkrll\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\",\n      \"name\": \"0000780c-679b-479e-b61c-e08211991339\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T17:27:09.508692971Z\",\n      \"endTime\": \"2026-09-18T17:51:11.117212687Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti77pkq7agiit4e9p66ic3479hnrq5 still exists (deletion dispatched at 2026-09-18T17:27:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019d60f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vmz65bf72428\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n      \"name\": \"e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T07:07:58.803605122Z\",\n      \"endTime\": \"2026-09-18T07:32:03.445509106Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95msfbb4nhua5a6pq8m3044dhe0se still exists (deletion dispatched at 2026-09-18T07:08:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019d60f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vmz65bf72428\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n      \"name\": \"e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T07:07:58.803605122Z\",\n      \"endTime\": \"2026-09-18T07:32:03.445509106Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95msfbb4nhua5a6pq8m3044dhe0se still exists (deletion dispatched at 2026-09-18T07:08:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c80150\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nglkjctcmb9p\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n      \"name\": \"9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T07:10:06.620770558Z\",\n      \"endTime\": \"2026-09-18T07:34:13.53272476Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95o86ftkou4io40i8o12g13uerlpd still exists (deletion dispatched at 2026-09-18T07:10:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c80150\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nglkjctcmb9p\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n      \"name\": \"9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T07:10:06.620770558Z\",\n      \"endTime\": \"2026-09-18T07:34:13.53272476Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95o86ftkou4io40i8o12g13uerlpd still exists (deletion dispatched at 2026-09-18T07:10:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"52f64f30d917c25349ded7085a1164f0e2615c20f54359a71e907e0ab61c033a","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","signature_id":"8db752092d73d2ebfaaaef7b1150f759438bf639db4c2c7645ebdce228946dcf","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","signature_id":"cc794ab286f7ef163ad70de687149ee27cd59dcc48d24d280de823796404177d","pr_number":6969,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f7e85048bd6764cc5537fb876507c8173ed1c8737be07b8c67fd9fcfdae58024","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,3,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001580078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dfbjv854d6w5\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\",\n      \"name\": \"d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:36:31.257109777Z\",\n      \"endTime\": \"2026-09-16T10:00:32.532474531Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss1375769r7k6ubu137jlbqtq9j35it still exists (deletion dispatched at 2026-09-16T09:36:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001580078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dfbjv854d6w5\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\",\n      \"name\": \"d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:36:31.257109777Z\",\n      \"endTime\": \"2026-09-16T10:00:32.532474531Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss1375769r7k6ubu137jlbqtq9j35it still exists (deletion dispatched at 2026-09-16T09:36:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011720a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-4cfzlz\" in resourcegroup=\"rg-zstream-upgrade-5-0-2bmmvg6nm4m6\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49e79c0-b564-4703-ad68-3e8fbb555ee7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49e79c0-b564-4703-ad68-3e8fbb555ee7\",\n      \"name\": \"a49e79c0-b564-4703-ad68-3e8fbb555ee7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:52:39.028633437Z\",\n      \"endTime\": \"2026-09-16T10:16:42.563449333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss136411ov65aeqqn1ut2t4iprrujv9 still exists (deletion dispatched at 2026-09-16T09:52:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011720a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-4cfzlz\" in resourcegroup=\"rg-zstream-upgrade-5-0-2bmmvg6nm4m6\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49e79c0-b564-4703-ad68-3e8fbb555ee7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49e79c0-b564-4703-ad68-3e8fbb555ee7\",\n      \"name\": \"a49e79c0-b564-4703-ad68-3e8fbb555ee7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:52:39.028633437Z\",\n      \"endTime\": \"2026-09-16T10:16:42.563449333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss136411ov65aeqqn1ut2t4iprrujv9 still exists (deletion dispatched at 2026-09-16T09:52:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010be048\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-lq45ld6h888d\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76270801-127d-4225-97dc-06e57322bf5a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76270801-127d-4225-97dc-06e57322bf5a\",\n      \"name\": \"76270801-127d-4225-97dc-06e57322bf5a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:37:08.667361714Z\",\n      \"endTime\": \"2026-09-16T10:01:12.766789046Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss13j30utlk9cm4u8n21tm903korv38 still exists (deletion dispatched at 2026-09-16T09:37:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010be048\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-lq45ld6h888d\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76270801-127d-4225-97dc-06e57322bf5a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76270801-127d-4225-97dc-06e57322bf5a\",\n      \"name\": \"76270801-127d-4225-97dc-06e57322bf5a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:37:08.667361714Z\",\n      \"endTime\": \"2026-09-16T10:01:12.766789046Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss13j30utlk9cm4u8n21tm903korv38 still exists (deletion dispatched at 2026-09-16T09:37:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"3ce6a891cb0d07c7ce901cd0ee33f19037e602ebe5639fc8eb520176e22be57f","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"5a758cc0e5fd4d4f0a2773989c77d2a7c459fe910fa5486bed83da0d068c3f31","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"70756564bd668b5819cf49c7d07af12de78c4ca05f5acb0024a436a269a2a249","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.273972602739726,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-q78vhx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f621e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-q78vhx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f621e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-prgsd6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106cb20\u003e: \n    DNS for route host agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-prgsd6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106cb20\u003e: \n    DNS for route host agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"803dd3435e0189d03ce01d4d7de3f8e741a5ee356c8001e19b84595bbaa71298","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524951428665344","occurred_at":"2026-09-17T09:59:04Z","signature_id":"b9c07fb6e82776d012cae86cfc979add2b4b19f7b344021874689105ee18c987","pr_number":5724,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c2687040eef152546e303fe3ce6da2f92a92db4c9a3989bfdcd2df869f68897f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.273972602739726,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f90b40\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8r8jd5phj8ps\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\",\n      \"name\": \"de938633-4e59-468a-9378-96c48d8ef3fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T03:48:43.242131083Z\",\n      \"endTime\": \"2026-09-18T04:17:44.681797406Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 16m33s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f90b40\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8r8jd5phj8ps\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\",\n      \"name\": \"de938633-4e59-468a-9378-96c48d8ef3fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T03:48:43.242131083Z\",\n      \"endTime\": \"2026-09-18T04:17:44.681797406Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 16m33s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-lzpm5g\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d707c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-lzpm5g: failed waiting for cluster=\"cp-ystream-upgrade-4-22-lzpm5g\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-v5f9s54xdgwx\" to finish creating: GET https://rp.j9719424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cff0006a-d905-424f-9efd-d8db75f2fd05\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cff0006a-d905-424f-9efd-d8db75f2fd05\",\n      \"name\": \"cff0006a-d905-424f-9efd-d8db75f2fd05\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:49:09.853286386Z\",\n      \"endTime\": \"2026-09-16T21:18:11.008528958Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 13m4s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-lzpm5g\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d707c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-lzpm5g: failed waiting for cluster=\"cp-ystream-upgrade-4-22-lzpm5g\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-v5f9s54xdgwx\" to finish creating: GET https://rp.j9719424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cff0006a-d905-424f-9efd-d8db75f2fd05\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cff0006a-d905-424f-9efd-d8db75f2fd05\",\n      \"name\": \"cff0006a-d905-424f-9efd-d8db75f2fd05\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:49:09.853286386Z\",\n      \"endTime\": \"2026-09-16T21:18:11.008528958Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 13m4s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","signature_id":"a6058215e786bb088f449c6f95b45c11ce369a5297dbcc0542779a89a2c66add","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309343839719424","occurred_at":"2026-09-16T19:42:24Z","signature_id":"af5ff2a9735a8e441c18465437503752dd1901dae01a546fbec14adb73b0c7e5","pr_number":6403,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.273972602739726,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,2,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019540a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-kttjnq\" in resourcegroup=\"rg-np-downgrade-kttjnq-75tphq26mtkt\" to finish deleting: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/941782dd-efae-4093-940b-ca21585fc901\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/941782dd-efae-4093-940b-ca21585fc901\",\n      \"name\": \"941782dd-efae-4093-940b-ca21585fc901\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:46:21.836683685Z\",\n      \"endTime\": \"2026-09-16T21:10:27.160876667Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssarr47iva7e9en1e12mbat1fvbbvpq still exists (deletion dispatched at 2026-09-16T20:46:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019540a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-kttjnq\" in resourcegroup=\"rg-np-downgrade-kttjnq-75tphq26mtkt\" to finish deleting: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/941782dd-efae-4093-940b-ca21585fc901\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/941782dd-efae-4093-940b-ca21585fc901\",\n      \"name\": \"941782dd-efae-4093-940b-ca21585fc901\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:46:21.836683685Z\",\n      \"endTime\": \"2026-09-16T21:10:27.160876667Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssarr47iva7e9en1e12mbat1fvbbvpq still exists (deletion dispatched at 2026-09-16T20:46:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001654180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-vwpvr2srntsx\" to finish deleting: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9690275d-f0a2-4269-97f3-ff42287a50a3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9690275d-f0a2-4269-97f3-ff42287a50a3\",\n      \"name\": \"9690275d-f0a2-4269-97f3-ff42287a50a3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:08:51.225823952Z\",\n      \"endTime\": \"2026-09-16T15:32:51.218856846Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss5qjn2q2n7cj829v2pum0bhr8rsof2 still exists (deletion dispatched at 2026-09-16T15:08:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001654180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-vwpvr2srntsx\" to finish deleting: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9690275d-f0a2-4269-97f3-ff42287a50a3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9690275d-f0a2-4269-97f3-ff42287a50a3\",\n      \"name\": \"9690275d-f0a2-4269-97f3-ff42287a50a3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:08:51.225823952Z\",\n      \"endTime\": \"2026-09-16T15:32:51.218856846Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss5qjn2q2n7cj829v2pum0bhr8rsof2 still exists (deletion dispatched at 2026-09-16T15:08:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"9cfc23c00b3b6ff4e037fcfa63ecc4ff6f6077b374d4b670e533df57b7d90b23","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"46b2146b5bea1fbbe7547a18b5b7e0a96d274edde0ce207e2338267f45f60742","pr_number":6976,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-8ld2f474grdt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-8ld2f474grdt, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.273972602739726,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["STG"],"trend_counts":[0,0,0,0,0,0,1,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00255de60\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-8ld2f474grdt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-8ld2f474grdt, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00255de60\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-8ld2f474grdt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-8ld2f474grdt, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.33\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017a97a0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-plrmgv\" resourcegroup=\"rg-np-version-upgrade-plrmgv-2td2q96dfsp2\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-plrmgv in resource group rg-np-version-upgrade-plrmgv-2td2q96dfsp2, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.33\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017a97a0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-plrmgv\" resourcegroup=\"rg-np-version-upgrade-plrmgv-2td2q96dfsp2\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-plrmgv in resource group rg-np-version-upgrade-plrmgv-2td2q96dfsp2, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101014543869153280","occurred_at":"2026-09-18T18:24:36Z","signature_id":"96d0afdee41d99182089b8922944bd77cbf90a6f0b4ca2fe3e238fbfdaae440e","pr_number":6639,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100487527256297472","occurred_at":"2026-09-17T07:30:18Z","signature_id":"e98964207b5fd40111448e87aa9fa8936a8a788c4325943a9b61b7b5870f98ff","pr_number":6952,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.273972602739726,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T21:40:09Z\n  Ended:       2026-09-17T21:58:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j1728768-svc\", component=\"backend\", name=\"externalauthclusterservicecreate\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue externalauthclusterservicecreate has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T15:28:20Z\n  Ended:       2026-09-16T15:40:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j8680064-svc\", component=\"backend\", name=\"dataplaneoidcfederation\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue dataplaneoidcfederation has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100680641501728768","occurred_at":"2026-09-17T20:17:48Z","signature_id":"f3ebbdf3ff2dce8e5f0f8347ed14b1024c81d64efecc73041c83a810441bae40","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"9efd97c6c35dce815008610c2dc8e01945d6be74819a30c6714b0f4b7f9ad9da","pr_number":6983,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.273972602739726,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":2}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-18T18:04:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1089664-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/services-j1089664\", namespace=\"klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5 on cluster ci01-j1089664-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-18T18:05:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1089664-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", lease=\"governance-policy-framework\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/services-j1089664\", namespace=\"klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease governance-policy-framework in namespace klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5 on cluster ci01-j1089664-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-18T18:05:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1089664-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", lease=\"config-policy-controller\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/services-j1089664\", namespace=\"klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease config-policy-controller in namespace klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5 on cluster ci01-j1089664-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:54:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j4807552-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.89:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4807552/providers/microsoft.monitor/accounts/services-j4807552\", namespace=\"klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-zvqnn\", prometheus=\"prometheus/prometheus\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn on cluster ci01-j4807552-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:54:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j4807552-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.89:8080\", job=\"kube-state-metrics\", lease=\"config-policy-controller\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4807552/providers/microsoft.monitor/accounts/services-j4807552\", namespace=\"klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-zvqnn\", prometheus=\"prometheus/prometheus\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease config-policy-controller in namespace klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn on cluster ci01-j4807552-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:54:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j4807552-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.89:8080\", job=\"kube-state-metrics\", lease=\"governance-policy-framework\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4807552/providers/microsoft.monitor/accounts/services-j4807552\", namespace=\"klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-zvqnn\", prometheus=\"prometheus/prometheus\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease governance-policy-framework in namespace klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn on cluster ci01-j4807552-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"b2d9d84ce6e1c74ea14279f086b91ae4fc06d24e40b82a7ad2aa54e78ffa7d4b","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"ed2b24c463c24323e023a9ff56bf6535b99fca0fd95a6e6a5f3b0c1f68b41036","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.273972602739726,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,2,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0014180c0\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-phbtk2 namespace are ready: failed to list pods in cilium-connectivity-check-phbtk2 namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0014180c0\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-phbtk2 namespace are ready: failed to list pods in cilium-connectivity-check-phbtk2 namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00168f0c8\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-txdwxr namespace are ready: failed to list pods in cilium-connectivity-check-txdwxr namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00168f0c8\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-txdwxr namespace are ready: failed to list pods in cilium-connectivity-check-txdwxr namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100434265043898368","occurred_at":"2026-09-17T03:58:47Z","signature_id":"297e17fd0f8b231f49a6a64a53967d8a3faf0348a5fdc0a0860997681f536656","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"2bd939aaad187ab78804671b6970c3fb300c0b1bc1be22955c06bc837de68bb9","pr_number":6976,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.273972602739726,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,2,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","occurrences":2}],"full_error_samples":["time=2026-09-16T10:35:02.273Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T10:35:02.716Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6352640-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6352640-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:ad132446-7529-417f-aff8-31d826ffb6f2 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#ad132446-7529-417f-aff8-31d826ffb6f2 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T10:35:02.716Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:35:02.863Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T10:35:03.042Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T10:35:03.042Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:35:03.143Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:45:05.949Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T10:45:05.949Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T10:46:05.955Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T10:46:06.637Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6352640-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6352640-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:ad132446-7529-417f-aff8-31d826ffb6f2 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#ad132446-7529-417f-aff8-31d826ffb6f2 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T10:46:06.637Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:07.013Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T10:46:07.408Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:07.408Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:07.670Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-16T10:46:08.488Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:08.489Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:08.888Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:09.401Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T10:46:09.747Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T10:46:09.959Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:10.361Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:10.729Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:11.065Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T10:46:11.065Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:11.148Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-16T10:56:12.973Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T10:56:12.973Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T10:57:12.977Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T10:57:13.486Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6352640-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6352640-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:ad132446-7529-417f-aff8-31d826ffb6f2 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#ad132446-7529-417f-aff8-31d826ffb6f2 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T10:57:13.486Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:13.807Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T10:57:14.174Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:14.174Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:14.404Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-16T10:57:15.329Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:15.329Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:15.688Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:15.987Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T10:57:16.345Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T10:57:16.535Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:16.747Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:16.984Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:17.208Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T10:57:17.208Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:17.300Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-16T11:05:02.358Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-16T11:05:02.359Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","time=2026-09-16T08:43:27.720Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T08:43:28.143Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6406656-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6406656-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:510fcd49-73a6-4643-b104-d1da046c7a87 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#510fcd49-73a6-4643-b104-d1da046c7a87 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T08:43:28.143Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:43:28.283Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T08:43:28.434Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T08:43:28.434Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:43:28.493Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:53:31.202Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T08:53:31.203Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T08:54:31.204Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T08:54:31.661Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6406656-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6406656-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:510fcd49-73a6-4643-b104-d1da046c7a87 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#510fcd49-73a6-4643-b104-d1da046c7a87 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T08:54:31.661Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:31.919Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T08:54:32.413Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:32.413Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:32.686Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-16T08:54:33.497Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:33.498Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:33.902Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:34.404Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T08:54:34.757Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T08:54:34.953Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:35.182Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:35.544Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:36.068Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T08:54:36.068Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:36.176Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-16T09:04:37.727Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T09:04:37.727Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T09:05:37.733Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T09:05:38.373Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6406656-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6406656-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:510fcd49-73a6-4643-b104-d1da046c7a87 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#510fcd49-73a6-4643-b104-d1da046c7a87 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T09:05:38.373Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:38.676Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T09:05:39.023Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:39.023Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:39.181Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-16T09:05:40.018Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:40.018Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:40.264Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:40.639Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T09:05:41.058Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T09:05:41.352Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:41.683Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:42.063Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:42.276Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T09:05:42.276Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:42.359Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-16T09:13:27.645Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-16T09:13:27.646Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161854826352640","occurred_at":"2026-09-16T09:56:19Z","signature_id":"64a318e95cdcfb12234888d476e960be3b284497df057c04c923477d134c5a2a","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132066686406656","occurred_at":"2026-09-16T07:57:56Z","signature_id":"d08b0cbd15edc9cffd195b1bd96feff56d9f329451cf7393af1293efbc3c0c96","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","failure_pattern":"private ingress should not be reachable from outside the VNet, but connection succeeded","search_query":"private ingress should not be reachable from outside the VNet, but connection succeeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.273972602739726,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,2,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred.  Got:\n    \u003cnil\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred.  Got:\n    \u003cnil\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100273649465430016","occurred_at":"2026-09-16T17:20:33Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252605216198656","occurred_at":"2026-09-16T15:56:55Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":6933,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,2,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-r67qjj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000814820\u003e: \n    DNS for route host agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-r67qjj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000814820\u003e: \n    DNS for route host agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-z9mjpt\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010173e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-z9mjpt\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010173e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252605216198656","occurred_at":"2026-09-16T15:56:55Z","signature_id":"662575cd71f8c3606ecf799e2d76b50f1b3ecb8c083205f994e08e750a8a40a2","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252605216198656","occurred_at":"2026-09-16T15:56:55Z","signature_id":"d9283aaa1d82489c4a0f2f12dd912bceba77f21d6bae60f6a616ab523d7ace51","pr_number":6933,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1d42c7dad9427d57e1b19754ffe767a1fc187725c1c8da2f03ad1f6733ce9b5d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,2,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aaaa60\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6mp8vkxcbh5s\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d35dfb81-23c2-4a18-80a3-7d5bf5893728\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d35dfb81-23c2-4a18-80a3-7d5bf5893728\",\n      \"name\": \"d35dfb81-23c2-4a18-80a3-7d5bf5893728\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T10:51:43.295429479Z\",\n      \"endTime\": \"2026-09-16T11:20:49.710317459Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aaaa60\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6mp8vkxcbh5s\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d35dfb81-23c2-4a18-80a3-7d5bf5893728\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d35dfb81-23c2-4a18-80a3-7d5bf5893728\",\n      \"name\": \"d35dfb81-23c2-4a18-80a3-7d5bf5893728\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T10:51:43.295429479Z\",\n      \"endTime\": \"2026-09-16T11:20:49.710317459Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011044c0\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-95l622wn4sm9\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b24629a-f978-4a3d-9d8d-8d5b6776590f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b24629a-f978-4a3d-9d8d-8d5b6776590f\",\n      \"name\": \"1b24629a-f978-4a3d-9d8d-8d5b6776590f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T10:52:02.869364535Z\",\n      \"endTime\": \"2026-09-16T11:21:09.719035401Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011044c0\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-95l622wn4sm9\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b24629a-f978-4a3d-9d8d-8d5b6776590f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b24629a-f978-4a3d-9d8d-8d5b6776590f\",\n      \"name\": \"1b24629a-f978-4a3d-9d8d-8d5b6776590f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T10:52:02.869364535Z\",\n      \"endTime\": \"2026-09-16T11:21:09.719035401Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"7299032a0750fba48148c45abb91daa5c3ae005efe59b617e7dc255e091f7a66","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"84f288aa90f1963969fa6df3b97105ac35a5dcb0b0f4f283ef7675cb82451eeb","pr_number":6791,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"3ec8dfede2b7d12b6200fcfa62f782082fd9d3f96c215ae2c1de00861966d250","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":2,"after_last_push_count":0,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d7ccc0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xp4rlvdpbgj6\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\",\n      \"name\": \"b344e921-2b47-4118-aac7-81c45cb67cc0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T06:13:19.863238771Z\",\n      \"endTime\": \"2026-09-18T06:42:22.545828006Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d7ccc0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xp4rlvdpbgj6\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\",\n      \"name\": \"b344e921-2b47-4118-aac7-81c45cb67cc0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T06:13:19.863238771Z\",\n      \"endTime\": \"2026-09-18T06:42:22.545828006Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a46a00\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jt9b8z9tj78s\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n      \"name\": \"16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T06:13:10.341418654Z\",\n      \"endTime\": \"2026-09-18T06:42:12.545110519Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a46a00\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jt9b8z9tj78s\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n      \"name\": \"16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T06:13:10.341418654Z\",\n      \"endTime\": \"2026-09-18T06:42:12.545110519Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","signature_id":"0d8835203476a034717107aadec991b28272c94910dad48bde3bb462bb56ac9b","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","signature_id":"4fb56cfa0c5d81c59f21dba90bd2b4aeaa2b18f5202205163a7a19b83e403f4e","pr_number":6157,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"240d7e65fa5e2b6ff5542fc888978c21e203da5524e2e297822471f59f51aa10","failure_pattern":"\"error\": {","search_query":"\"error\": {","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:139]: failed to get backup schedules for cadence check\nUnexpected error:\n    \u003c*errors.errorString | 0xc0010c3e40\u003e: \n    expected status 200, got 412: {\n        \"error\": {\n            \"code\": \"InvalidResource\",\n            \"message\": \"kube-applier client not available for management cluster /providers/microsoft.redhatopenshift/stamps/1/managementclusters/default\"\n        }\n    }\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:139]: failed to get backup schedules for cadence check\nUnexpected error:\n    \u003c*errors.errorString | 0xc0010c3e40\u003e: \n    expected status 200, got 412: {\n        \"error\": {\n            \"code\": \"InvalidResource\",\n            \"message\": \"kube-applier client not available for management cluster /providers/microsoft.redhatopenshift/stamps/1/managementclusters/default\"\n        }\n    }\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","signature_id":"06ba3d2a9582d2a18869bdd5ae1df4321a4d0977a7be4bdd0e3cc1af2ec0608a","pr_number":6157,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-update-nodes: failed waiting for nodepool=\"np-update-nodes\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-ckch4xg6nxh4\" to finish creating: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e55eeddb-0ead-432e-a110-c696080abf84\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e55eeddb-0ead-432e-a110-c696080abf84\",\n  \"name\": \"e55eeddb-0ead-432e-a110-c696080abf84\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-16T20:39:13.615216045Z\",\n  \"endTime\": \"2026-09-16T20:58:16.977517435Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"node pool creation did not complete before the deadline\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-update-nodes: failed waiting for nodepool=\"np-update-nodes\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-ckch4xg6nxh4\" to finish creating: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e55eeddb-0ead-432e-a110-c696080abf84\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e55eeddb-0ead-432e-a110-c696080abf84\",\n  \"name\": \"e55eeddb-0ead-432e-a110-c696080abf84\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-16T20:39:13.615216045Z\",\n  \"endTime\": \"2026-09-16T20:58:16.977517435Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"node pool creation did not complete before the deadline\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"663948fad2f655a8f86778686b2675b46d2a717e3dc1afdecbbc520028b3d155","pr_number":6945,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f59e236e067df212a87e97ef00e5cb14f86f6db93d160e4c5cd41734c06ea1b3","failure_pattern":"Admin credentials did not return 409 conflict error while cluster is deploying","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:144]: Admin credentials did not return 409 conflict error while cluster is deploying","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:144]: Admin credentials did not return 409 conflict error while cluster is deploying\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:144]: Admin credentials did not return 409 conflict error while cluster is deploying"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100237146764873728","occurred_at":"2026-09-16T14:55:30Z","signature_id":"6628b29b597aa4c3bb50125f87195e6a58208831ab9330e6078d3d1c523d7af7","pr_number":6752,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4f70c086dc5495ea728fc39aab23a54b6cce4072ba8ccfc36ee6980300ba5299","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-j7tx89s2rwk5\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-j7tx89s2rwk5, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:276]: failed to create external auth config \"ext-auth-cidr\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002990240\u003e: \n    failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-j7tx89s2rwk5\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-j7tx89s2rwk5, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:276]: failed to create external auth config \"ext-auth-cidr\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002990240\u003e: \n    failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-j7tx89s2rwk5\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-j7tx89s2rwk5, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100680641501728768","occurred_at":"2026-09-17T20:17:48Z","signature_id":"bf84ee4449e687d0ad4cd4859ef7515e9901f5ded80364d34ebb748aaa5b41df","pr_number":6999,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a16108\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a16108\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"e37087f31e7094fe45c7ae2868faf05fde32666577b47902fe94fbe270287c3b","pr_number":6993,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"377edc68350e3289212c2970b00ba8365c9d949a03379484101a64858ed9715c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message The collection with name [Resources] in provided scope [/dbs/\u003ccosmos-account\u003e/colls/Resources] could not be found.\\\\\\ \\\\ ActivityId: \u003cuuid\u003e, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0; provider Microsoft.DocumentDB","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1}],"full_error_samples":["time=2026-09-16T13:31:49.803Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-09-16T13:31:51.360Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-09-16T13:31:52.622Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=6a654e03e0d8cd3ba2dcf4b7ca7ea7b9161da4e2734cccce092b3f0e303e2fe9 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6373376-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6a654e03e0d8cd3ba2dcf4b7ca7ea7b9161da4e2734cccce092b3f0e303e2fe9\ntime=2026-09-16T13:40:24.258Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6373376-svc/providers/Microsoft.Resources/deployments/6a654e03e0d8cd3ba2dcf4b7ca7ea7b9161da4e2734cccce092b3f0e303e2fe9/operationStatuses/08584120413735528505\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6373376/providers/Microsoft.Resources/deployments/rp_cosmos_db\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6373376/providers/Microsoft.DocumentDB/databaseAccounts/arohcpci01-rp-j6373376/sqlRoleAssignments/3a943d25-7e89-52ef-bdd2-d3814561859c\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The collection with name [Resources] in provided scope [/dbs/arohcpci01-rp-j6373376/colls/Resources] could not be found.\\\\\\\\r\\\\\\\\nActivityId: 385582e2-ce0e-4904-a9f2-7469a3089fa3, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100209889606373376","occurred_at":"2026-09-16T13:07:12Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6886,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"54082ad8881c5131bff996f7153979c81faa7fa74e3b195d969ebc6eb63a2c19","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/cluster-\u003cid\u003e' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '\u003ctimestamp\u003e' with correlationId '\u003cuuid\u003e', and will expire at '\u003ctimestamp\u003e' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-09-17T08:52:09.303Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T08:52:10.143Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-17T08:52:11.020Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=e5c452463d5a2ca118d246d0e10c4f76fe32f1478279da2b4450028ca1d7e3f8 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2728320-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fe5c452463d5a2ca118d246d0e10c4f76fe32f1478279da2b4450028ca1d7e3f8\ntime=2026-09-17T08:54:11.408Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2728320-mgmt-1/providers/Microsoft.Resources/deployments/e5c452463d5a2ca118d246d0e10c4f76fe32f1478279da2b4450028ca1d7e3f8/operationStatuses/08584119717548817375\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentActive\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2728320-mgmt-1/providers/Microsoft.Resources/deployments/cluster-34wn2l5qqmupi' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '9/17/2026 8:52:28 AM' with correlationId 'd7f46ae1-ae89-4906-81b7-ab9a15265766', and will expire at '9/24/2026 8:52:28 AM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100500788332728320","occurred_at":"2026-09-17T08:23:07Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0008473d183c6e76249427529630789e6d4f9e8f1d5bdbea4b6bd57ae83c8f06","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/svc-kv-xehipxbjxawki' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '\u003ctimestamp\u003e' with correlationId '\u003cuuid\u003e', and will expire at '\u003ctimestamp\u003e' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra-output","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra-output","occurrences":1}],"full_error_samples":["time=2026-09-16T08:27:04.451Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output description=\"Step infra-output\\n  Kind: ARM\\n  Template: templates/svc-infra-lookup.bicep\\n  Parameters: configurations/svc-infra-lookup.tmpl.bicepparam\"\ntime=2026-09-16T08:27:04.849Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output\ntime=2026-09-16T08:27:05.446Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output deployment=eaaad5ae26368062d8da5183fba1f1cf527c2665c3e53af22a660fa55c8dc157 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7918592-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Feaaad5ae26368062d8da5183fba1f1cf527c2665c3e53af22a660fa55c8dc157\ntime=2026-09-16T08:27:35.588Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7918592-svc/providers/Microsoft.Resources/deployments/eaaad5ae26368062d8da5183fba1f1cf527c2665c3e53af22a660fa55c8dc157/operationStatuses/08584120596603628303\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentActive\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/svc-kv-xehipxbjxawki' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '9/16/2026 8:27:06 AM' with correlationId '032c9251-9fc6-4ff2-b8e3-30d15533261b', and will expire at '9/23/2026 8:27:06 AM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132012407918592","occurred_at":"2026-09-16T07:57:44Z","signature_id":"09b1458b46da16205cd209ec402488a728d06d1e258b36845618ba3741f96692","pr_number":6639,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"8aa573d68ede374d56fcd25dace5feffa8644835d49bf0c7ba52006e6de9139d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentNotFound; detail message Deployment 'svc-kv-xehipxbjxawki' could not be found.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra-output","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra-output","occurrences":1}],"full_error_samples":["time=2026-09-18T18:49:43.745Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output description=\"Step infra-output\\n  Kind: ARM\\n  Template: templates/svc-infra-lookup.bicep\\n  Parameters: configurations/svc-infra-lookup.tmpl.bicepparam\"\ntime=2026-09-18T18:49:44.412Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output\ntime=2026-09-18T18:49:45.666Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output deployment=299b33c33a8d1927824771d1411d9cc2911c3c4239fa63e4af7c6c883950aabd portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6366720-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F299b33c33a8d1927824771d1411d9cc2911c3c4239fa63e4af7c6c883950aabd\ntime=2026-09-18T18:50:16.326Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6366720-svc/providers/Microsoft.Resources/deployments/299b33c33a8d1927824771d1411d9cc2911c3c4239fa63e4af7c6c883950aabd/operationStatuses/08584118495004368821\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"NotFound\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentNotFound\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Deployment 'svc-kv-xehipxbjxawki' could not be found.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7017/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101014245486366720","occurred_at":"2026-09-18T18:23:12Z","signature_id":"09b1458b46da16205cd209ec402488a728d06d1e258b36845618ba3741f96692","pr_number":7017,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ef6114015f72fab4d7dec149451d78cb6b761172f511b8fdedd7928043056bed","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ErrCode_InsufficientVCPUQuota; detail message Insufficient vcpu quota requested \u003ccount\u003e, remaining \u003ccount\u003e for family StandardEdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: \u003curl\u003e Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-16T15:02:01.563Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-16T15:02:02.231Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-16T15:02:03.235Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=9b458a858b62ced8b861c50221db722ca7b84af5c0247e7866fc1bf068dbe8ac portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7202944-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9b458a858b62ced8b861c50221db722ca7b84af5c0247e7866fc1bf068dbe8ac\ntime=2026-09-16T15:11:05.045Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.Resources/deployments/9b458a858b62ced8b861c50221db722ca7b84af5c0247e7866fc1bf068dbe8ac/operationStatuses/08584120359627896142\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.Resources/deployments/cluster-oqca7h7ulqwlw\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7202944-mgmt-2/agentPools/userswft3\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 96, remaining 48 for family StandardEdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n                  },\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7202944-mgmt-2/agentPools/userswft2\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 96, remaining 48 for family StandardEdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n                  },\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7202944-mgmt-2/agentPools/userswft1\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 96, remaining 48 for family StandardEdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100233505077202944","occurred_at":"2026-09-16T14:41:02Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6980,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fe24c8b4b28d3d2b1937803a92704c44ec574f3d1c3b740ec6d97282a1e18d39","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00230a300\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ql5j7w6fx6vn\" to finish creating: GET https://rp.j6364288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5fde05ba-68c2-437a-84ea-82765b356728\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5fde05ba-68c2-437a-84ea-82765b356728\",\n      \"name\": \"5fde05ba-68c2-437a-84ea-82765b356728\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T11:04:09.351110934Z\",\n      \"endTime\": \"2026-09-16T11:04:15.302518412Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00230a300\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ql5j7w6fx6vn\" to finish creating: GET https://rp.j6364288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5fde05ba-68c2-437a-84ea-82765b356728\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5fde05ba-68c2-437a-84ea-82765b356728\",\n      \"name\": \"5fde05ba-68c2-437a-84ea-82765b356728\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T11:04:09.351110934Z\",\n      \"endTime\": \"2026-09-16T11:04:15.302518412Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100164075236364288","occurred_at":"2026-09-16T10:05:09Z","signature_id":"59512808dbcb20b06030c9fde07924554878fee722f13b2a06c73ba1d94bd2f9","pr_number":6688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6ed58550ec6b76673ce00588c8bebee5ecb6ea4c774ba1ceae5e31e8e80cd829","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000edc320\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-d77hgbbbw6rf\" to finish creating: GET https://rp.j6676096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74e66356-25b3-4d6f-95af-f1292008b6fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74e66356-25b3-4d6f-95af-f1292008b6fd\",\n      \"name\": \"74e66356-25b3-4d6f-95af-f1292008b6fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:47:22.69341748Z\",\n      \"endTime\": \"2026-09-16T21:16:24.116227211Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 6m20s ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000edc320\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-d77hgbbbw6rf\" to finish creating: GET https://rp.j6676096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74e66356-25b3-4d6f-95af-f1292008b6fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74e66356-25b3-4d6f-95af-f1292008b6fd\",\n      \"name\": \"74e66356-25b3-4d6f-95af-f1292008b6fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:47:22.69341748Z\",\n      \"endTime\": \"2026-09-16T21:16:24.116227211Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 6m20s ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100314659356676096","occurred_at":"2026-09-16T20:03:31Z","signature_id":"ec03acc423d7d391cd0db46f80f6e282dd4d144bd87644aded93eda8c1bf31c4","pr_number":6969,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"46425ffc9312d96c5106d1b27946c7775e4ddf6d0471e792a3aaf8c2bc4d3a86","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbc120\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-w5shsm9ng8kc\" to finish creating: GET https://rp.j5021696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\",\n      \"name\": \"eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T04:52:38.625826904Z\",\n      \"endTime\": \"2026-09-17T05:21:45.494759028Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 2m17s ago; version 4.22.14 is Partial (want Completed), started 5m22s ago; version 4.22.14 is Partial (want Completed), started 12m16s ago; version 4.22.14 is Partial (want Completed), started 15m20s ago; version 4.22.14 is Partial (want Completed), started 17m23s ago; version 4.22.14 is Partial (want Completed), started 17m58s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbc120\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-w5shsm9ng8kc\" to finish creating: GET https://rp.j5021696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\",\n      \"name\": \"eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T04:52:38.625826904Z\",\n      \"endTime\": \"2026-09-17T05:21:45.494759028Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 2m17s ago; version 4.22.14 is Partial (want Completed), started 5m22s ago; version 4.22.14 is Partial (want Completed), started 12m16s ago; version 4.22.14 is Partial (want Completed), started 15m20s ago; version 4.22.14 is Partial (want Completed), started 17m23s ago; version 4.22.14 is Partial (want Completed), started 17m58s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100438339525021696","occurred_at":"2026-09-17T04:14:58Z","signature_id":"421f5b6d838c121843fe235c14d4d70af34427ca212e0849c68ed8aa29654d6b","pr_number":6986,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"5d59b4e4feb4fd290a5c1b398014539407c812b82baa8f933608971503b0bc0d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8a120\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-8prbhvsvlkq9\" to finish creating: GET https://rp.j9922816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b148abb4-8011-43a5-acf9-7d2f1f124b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b148abb4-8011-43a5-acf9-7d2f1f124b53\",\n      \"name\": \"b148abb4-8011-43a5-acf9-7d2f1f124b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:56:50.517658972Z\",\n      \"endTime\": \"2026-09-16T21:25:54.975318736Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8a120\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-8prbhvsvlkq9\" to finish creating: GET https://rp.j9922816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b148abb4-8011-43a5-acf9-7d2f1f124b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b148abb4-8011-43a5-acf9-7d2f1f124b53\",\n      \"name\": \"b148abb4-8011-43a5-acf9-7d2f1f124b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:56:50.517658972Z\",\n      \"endTime\": \"2026-09-16T21:25:54.975318736Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309676229922816","occurred_at":"2026-09-16T19:43:43Z","signature_id":"a48d7255fe4f2e537c6260d9fc791f5067ce6859f69937f424d6c26f5bc86ca9","pr_number":6894,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"676e68f8a68616ac8c57dc0fe075f4a0d00c211dff572a92461d02984b57a771","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098aa40\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-6th5jd8nbg5s\" to finish creating: GET https://rp.j2424320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd1a9666-9126-4a24-9b9a-2799ae9c069b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd1a9666-9126-4a24-9b9a-2799ae9c069b\",\n      \"name\": \"bd1a9666-9126-4a24-9b9a-2799ae9c069b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T13:57:52.148119457Z\",\n      \"endTime\": \"2026-09-16T14:26:53.36802381Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, community-operators-catalog, hosted-cluster-config-operator, catalog-operator, redhat-operators-catalog, ingress-operator, cluster-image-registry-operator, openshift-route-controller-manager, cluster-node-tuning-operator, dns-operator, redhat-marketplace-catalog, certified-operators-catalog, cluster-network-operator, packageserver, openshift-controller-manager, cluster-policy-controller, cluster-version-operator, cluster-storage-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098aa40\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-6th5jd8nbg5s\" to finish creating: GET https://rp.j2424320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd1a9666-9126-4a24-9b9a-2799ae9c069b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd1a9666-9126-4a24-9b9a-2799ae9c069b\",\n      \"name\": \"bd1a9666-9126-4a24-9b9a-2799ae9c069b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T13:57:52.148119457Z\",\n      \"endTime\": \"2026-09-16T14:26:53.36802381Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, community-operators-catalog, hosted-cluster-config-operator, catalog-operator, redhat-operators-catalog, ingress-operator, cluster-image-registry-operator, openshift-route-controller-manager, cluster-node-tuning-operator, dns-operator, redhat-marketplace-catalog, certified-operators-catalog, cluster-network-operator, packageserver, openshift-controller-manager, cluster-policy-controller, cluster-version-operator, cluster-storage-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100212360202424320","occurred_at":"2026-09-16T13:16:57Z","signature_id":"fec53f2216f9f1f9da9e1788f3642b62d62882fc08fe207f753fbff13e9390bf","pr_number":6894,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0004ec1e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-hdsrgp87vhp6\" to finish deleting: GET https://rp.j6073984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/140fc1a6-e41e-4f25-9908-25d174a04842\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/140fc1a6-e41e-4f25-9908-25d174a04842\",\n      \"name\": \"140fc1a6-e41e-4f25-9908-25d174a04842\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T21:51:08.271574522Z\",\n      \"endTime\": \"2026-09-17T22:15:12.38485138Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st0r3e7gsp00p1v6je2veqpisj6tv8b still exists (deletion dispatched at 2026-09-17T21:51:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0004ec1e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-hdsrgp87vhp6\" to finish deleting: GET https://rp.j6073984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/140fc1a6-e41e-4f25-9908-25d174a04842\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/140fc1a6-e41e-4f25-9908-25d174a04842\",\n      \"name\": \"140fc1a6-e41e-4f25-9908-25d174a04842\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T21:51:08.271574522Z\",\n      \"endTime\": \"2026-09-17T22:15:12.38485138Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st0r3e7gsp00p1v6je2veqpisj6tv8b still exists (deletion dispatched at 2026-09-17T21:51:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685654236073984","occurred_at":"2026-09-17T20:37:43Z","signature_id":"fa245128857c23599fcf8f3484b06ab6ca5868cca2b19c36072deb31d3760300","pr_number":7000,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"82fcc2cca51beebf00c7302e8f8dc3563b7995a4550dbcf53d6d84d1820ed342","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00101a2b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-fth2ns8rql9z\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8997c6b-6d2a-4f02-afe2-3927067cda87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8997c6b-6d2a-4f02-afe2-3927067cda87\",\n      \"name\": \"e8997c6b-6d2a-4f02-afe2-3927067cda87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T09:27:47.803109699Z\",\n      \"endTime\": \"2026-09-17T09:51:54.618583456Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sslv6ma6mn5k5mt36luuom09289pq58 still exists (deletion dispatched at 2026-09-17T09:27:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00101a2b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-fth2ns8rql9z\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8997c6b-6d2a-4f02-afe2-3927067cda87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8997c6b-6d2a-4f02-afe2-3927067cda87\",\n      \"name\": \"e8997c6b-6d2a-4f02-afe2-3927067cda87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T09:27:47.803109699Z\",\n      \"endTime\": \"2026-09-17T09:51:54.618583456Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sslv6ma6mn5k5mt36luuom09289pq58 still exists (deletion dispatched at 2026-09-17T09:27:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"724c2a9602b01afbc546852f539ff27b105da0871a0374f448eae350e85e32ce","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a9f77fe91a65ae001825b7e195168037cff1cc1fd2f1b2a967813a5f71db56d2","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f92288\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-djgr48qrs9vr\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\",\n      \"name\": \"a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:36:21.892022588Z\",\n      \"endTime\": \"2026-09-16T10:00:22.50023585Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss12t13cvqgvb69duuok3m0c1if4g29 still exists (deletion dispatched at 2026-09-16T09:36:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f92288\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-djgr48qrs9vr\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\",\n      \"name\": \"a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:36:21.892022588Z\",\n      \"endTime\": \"2026-09-16T10:00:22.50023585Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss12t13cvqgvb69duuok3m0c1if4g29 still exists (deletion dispatched at 2026-09-16T09:36:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"f25ee5738903cfa5e80de14eb219fe7a6033dded2da21704c137087674937bf8","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b2d9bdf10bd756a41af895df7f16c38d9b8ca7729eae2184e9c4375fba7db5fb","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012b4030\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-sk5lk2xqrw7v\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/234e26dc-5c76-4335-9749-4a1c49aae81e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/234e26dc-5c76-4335-9749-4a1c49aae81e\",\n      \"name\": \"234e26dc-5c76-4335-9749-4a1c49aae81e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:00:40.136241522Z\",\n      \"endTime\": \"2026-09-16T17:24:42.83613595Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7is141l1pe3gtmk90b2cej7ltp3gn still exists (deletion dispatched at 2026-09-16T17:00:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012b4030\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-sk5lk2xqrw7v\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/234e26dc-5c76-4335-9749-4a1c49aae81e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/234e26dc-5c76-4335-9749-4a1c49aae81e\",\n      \"name\": \"234e26dc-5c76-4335-9749-4a1c49aae81e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:00:40.136241522Z\",\n      \"endTime\": \"2026-09-16T17:24:42.83613595Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7is141l1pe3gtmk90b2cej7ltp3gn still exists (deletion dispatched at 2026-09-16T17:00:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"5cf9c85c408d35be4deef4e7c087c454f1070d1509c8652940e0ebeb5471037c","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f3bc88c6c8876f1c2ea5668dd924adc7c81d6e0349b9850342e67442ed3fe914","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00108c078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mzfrqt8tf9wq\" to finish deleting: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\",\n      \"name\": \"e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T11:18:42.744808437Z\",\n      \"endTime\": \"2026-09-16T11:42:49.617012035Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00108c078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mzfrqt8tf9wq\" to finish deleting: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\",\n      \"name\": \"e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T11:18:42.744808437Z\",\n      \"endTime\": \"2026-09-16T11:42:49.617012035Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"874e6ea76f2ba4acc89e73ac761bf0c81c038b75f11db73023de66c025986ab7","pr_number":6791,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0015da030\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2vshj8kd4ngg\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b25208f-0e8c-46c2-96d1-92647ed090d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b25208f-0e8c-46c2-96d1-92647ed090d2\",\n      \"name\": \"2b25208f-0e8c-46c2-96d1-92647ed090d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T09:42:31.310516975Z\",\n      \"endTime\": \"2026-09-17T10:06:34.577233142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0015da030\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2vshj8kd4ngg\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b25208f-0e8c-46c2-96d1-92647ed090d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b25208f-0e8c-46c2-96d1-92647ed090d2\",\n      \"name\": \"2b25208f-0e8c-46c2-96d1-92647ed090d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T09:42:31.310516975Z\",\n      \"endTime\": \"2026-09-17T10:06:34.577233142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"b8fb61f35748687f1492367050e150cb136907cb120fe4d9a22bd956f2ad9b80","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"27391593c89e1b6a3153b058ef31be166dd830240fb93e3caa733a62c02edc05","failure_pattern":"ERROR CODE: ResourceCollectionRequestsThrottled; provider Microsoft.Network","search_query":"ERROR CODE: ResourceCollectionRequestsThrottled","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:141]: failed to find private KAS internal LB IP in managed resource group \"private-kas-pzzdfb6jcgvd--managed\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001100480\u003e: \n    failed to list load balancers in \"private-kas-pzzdfb6jcgvd--managed\": GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-pzzdfb6jcgvd--managed/providers/Microsoft.Network/loadBalancers\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: ResourceCollectionRequestsThrottled\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceCollectionRequestsThrottled\",\n        \"message\": \"Operation 'Microsoft.Network/loadBalancers/read' failed as server encountered too many requests. Please try after '1' seconds. Tracking Id is '0bca3dc6-65f8-40d2-be6e-d4168ed4b62f'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:141]: failed to find private KAS internal LB IP in managed resource group \"private-kas-pzzdfb6jcgvd--managed\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001100480\u003e: \n    failed to list load balancers in \"private-kas-pzzdfb6jcgvd--managed\": GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-pzzdfb6jcgvd--managed/providers/Microsoft.Network/loadBalancers\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: ResourceCollectionRequestsThrottled\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceCollectionRequestsThrottled\",\n        \"message\": \"Operation 'Microsoft.Network/loadBalancers/read' failed as server encountered too many requests. Please try after '1' seconds. Tracking Id is '0bca3dc6-65f8-40d2-be6e-d4168ed4b62f'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488796448493568","occurred_at":"2026-09-17T07:35:28Z","signature_id":"dcc1d91a022ed0888281ddf9849c28aaaf8b8a73e6c45ef734d0a5c17a44b021","pr_number":6856,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--rrwf84jdrq9t/cluster-hs-pre-dlzcc5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007ce420\u003e: \n    failed to create HCP cluster cluster-hs-pre-dlzcc5: failed waiting for cluster=\"cluster-hs-pre-dlzcc5\" in resourcegroup=\"rg-hs-pre--rrwf84jdrq9t\" to finish creating: GET https://rp.j1442176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--rrwf84jdrq9t/cluster-hs-pre-dlzcc5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007ce420\u003e: \n    failed to create HCP cluster cluster-hs-pre-dlzcc5: failed waiting for cluster=\"cluster-hs-pre-dlzcc5\" in resourcegroup=\"rg-hs-pre--rrwf84jdrq9t\" to finish creating: GET https://rp.j1442176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100735274731442176","occurred_at":"2026-09-17T23:54:53Z","signature_id":"a9eee6e574575c9254a566a1c15978b6cdd24af6e7e61c298d051c731edd627c","pr_number":6939,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"5cfe4f61cfcfa6903321f1c971ea5ab31ebee2ed754015e96c99b68dfe286ebe","failure_pattern":"VM run command \"e2e-runcommand-1789704441845701237\" on VM \"\u003cvm\u003e\" failed and cleanup also failed: Get \"\u003curl\u003e\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"\u003curl\u003e\": stream error: stream ID 33; CANCEL; received from peer)","search_query":"VM run command \"e2e-runcommand-1789704441845701237\" on VM \"cidr-connectivity-test-test-vm\" failed and cleanup also failed: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/locations/CanadaCentral/operations/67651689-75ab-4f6c-aa6e-1746fbbdc2ad?p=c6bb2de4-260b-49a6-86a7-56c69bdc0403\u0026api-version=2024-03-01\u0026t=639253012427558463\u0026c=MIIHlDCCBnygAwIBAgIQTJW7e7XbMoQlWPa331WcnDANBgkqhkiG9w0BAQsFADA2MTQwMgYDVQQDEytDQ01FIEcxIFRMUyBSU0EgMjA0OCBTSEEyNTYgMjA0OSBXVVMyIENBIDAxMB4XDTI2MDgxMTE3MzMyMloXDTI3MDIwNjIzMzMyMlowQDE-MDwGA1UEAxM1YXN5bmNvcGVyYXRpb25zaWduaW5nY2VydGlmaWNhdGUubWFuYWdlbWVudC5henVyZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeQHh5tmZ6ZYRlf_Q_PwPJqUN9t1MEgnet-7jMlsIvkZBhjMDVPNEFS1UtIM-3StxxWCffcghG31P1UcexhHelJCujRWSZYWfEIC7z_S8BBwxDhcc2peEIdNKD1O6UKvglkD8KRsPxRT-ZwQYY3qFNngixnx3EDhtWr1a9Kg46bdJJAIIzu3flYJzQzP3yVrFuRQn4iH75UvhgEByfjJaD75Ugw9JYkgO52nE5dVc2ddxoRMWzJKxp_V2Adp7VqTbMx2T8hO9PWqkltIPwoAXIubXFj3Z5A6Wj7hE7XmY_2cFJKJfspjiqwLyfnLqlv9pCLhIVdG31g8DAGZM7fc2lAgMBAAGjggSSMIIEjjCBnQYDVR0gBIGVMIGSMAwGCisGAQQBgjd7AQEwZgYKKwYBBAGCN3sCAjBYMFYGCCsGAQUFBwICMEoeSAAzADMAZQAwADEAOQAyADEALQA0AGQANgA0AC0ANABmADgAYwAtAGEAMAA1ADUALQA1AGIAZABhAGYAZgBkADUAZQAzADMAZDAMBgorBgEEAYI3ewMCMAwGCisGAQQBgjd7BAIwDAYDVR0TAQH_BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDgYDVR0PAQH_BAQDAgWgMB0GA1UdDgQWBBQbwB3CifQh36XI9i1lBe8503G1RTAfBgNVHSMEGDAWgBSs43L6A7Jznj2VyO-HW67dG6HtaDCCAbIGA1UdHwSCAakwggGlMGmgZ6BlhmNodHRwOi8vcHJpbWFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwa6BpoGeGZWh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3R1czIvY3Jscy9jY21ld2VzdHVzMnBraS9jY21ld2VzdHVzMmljYTAxLzY3L2N1cnJlbnQuY3JsMFqgWKBWhlRodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwb6BtoGuGaWh0dHA6Ly9jY21ld2VzdHVzMnBraS53ZXN0dXMyLnBraS5jb3JlLndpbmRvd3MubmV0L2NlcnRpZmljYXRlQXV0aG9yaXRpZXMvY2NtZXdlc3R1czJpY2EwMS82Ny9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jYWNlcnRzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWV3ZXN0dXMycGtpLndlc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQACHqTwaTHdHWv9uHzVkq0ltJxzjLPIQHJnP6ndMrFnY-TIxQDrABl7eSPlINC2qk1CFExKLmjYM8aDst83BXP-yH0AHIvBrOGVdZvqFE-8KPpkqE97J5NNLNr1lLkz4vWpbjmJhLvYmureoMJjtcLpQyz-EfACVM90Lkmu8ueIkp5mjzxa_-cMa3KYwbFXLO8huXYcrhLhvs31Z9czJKjlPptL0ZE1-rBlMXGqTdZUE1CfPRoTjMMRZe_jEJYNKyxhL_5fT2mLxdlYJsMDeTU5qM-x3lA74zxVrUvC0ZvCQdK629aZQUbWV74Uzk1DmAtUiDhGvnsOW-rug_uLr621\u0026s=2ED070N0ZTuw-f1HVDJ70D4TL-E7a_pmemS26JAcsDWdtZNt5piI-6t7rp4DAeUvveysVwPzvAf5ULh-g26b96AAV9pXZ4I1vo74l2qjDBTA1ho2Ia11r_3U1XAtiPoLjGufya8CYXOek1iwZu7oaFEZ-4uayFIei-jUkQWxZ9Jz8_HKfyzJZZA1qJtJWKMIu_-fMQRELNXCWOkBgzgrUj1MP39GcY4oa5lZoVmPJGhHGseHZ--ETS4Vun62lwfxl7XrPb2PN0L9g7S07Nil12FIWSbh_qVJ2DC5m-7fPlkYze7LvocmWvHPwMOLiIbvvBS9Ui4BX7syQlC25zGKXw\u0026h=siGPFN1sMGFTEzhJalRHQiDtbRbqB8eZGXQRBJUWkyo\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qhh4b47m8ncv/providers/Microsoft.Compute/virtualMachines/cidr-connectivity-test-test-vm/runCommands/e2e-runcommand-1789704441845701237?api-version=2024-03-01\": stream error: stream ID 33; CANCEL; received from peer)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:293]: failed to create cluster role binding for external auth via VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d173e0\u003e: \n    VM run command \"e2e-runcommand-1789704441845701237\" on VM \"cidr-connectivity-test-test-vm\" failed and cleanup also failed: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/locations/CanadaCentral/operations/67651689-75ab-4f6c-aa6e-1746fbbdc2ad?p=c6bb2de4-260b-49a6-86a7-56c69bdc0403\u0026api-version=2024-03-01\u0026t=639253012427558463\u0026c=MIIHlDCCBnygAwIBAgIQTJW7e7XbMoQlWPa331WcnDANBgkqhkiG9w0BAQsFADA2MTQwMgYDVQQDEytDQ01FIEcxIFRMUyBSU0EgMjA0OCBTSEEyNTYgMjA0OSBXVVMyIENBIDAxMB4XDTI2MDgxMTE3MzMyMloXDTI3MDIwNjIzMzMyMlowQDE-MDwGA1UEAxM1YXN5bmNvcGVyYXRpb25zaWduaW5nY2VydGlmaWNhdGUubWFuYWdlbWVudC5henVyZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeQHh5tmZ6ZYRlf_Q_PwPJqUN9t1MEgnet-7jMlsIvkZBhjMDVPNEFS1UtIM-3StxxWCffcghG31P1UcexhHelJCujRWSZYWfEIC7z_S8BBwxDhcc2peEIdNKD1O6UKvglkD8KRsPxRT-ZwQYY3qFNngixnx3EDhtWr1a9Kg46bdJJAIIzu3flYJzQzP3yVrFuRQn4iH75UvhgEByfjJaD75Ugw9JYkgO52nE5dVc2ddxoRMWzJKxp_V2Adp7VqTbMx2T8hO9PWqkltIPwoAXIubXFj3Z5A6Wj7hE7XmY_2cFJKJfspjiqwLyfnLqlv9pCLhIVdG31g8DAGZM7fc2lAgMBAAGjggSSMIIEjjCBnQYDVR0gBIGVMIGSMAwGCisGAQQBgjd7AQEwZgYKKwYBBAGCN3sCAjBYMFYGCCsGAQUFBwICMEoeSAAzADMAZQAwADEAOQAyADEALQA0AGQANgA0AC0ANABmADgAYwAtAGEAMAA1ADUALQA1AGIAZABhAGYAZgBkADUAZQAzADMAZDAMBgorBgEEAYI3ewMCMAwGCisGAQQBgjd7BAIwDAYDVR0TAQH_BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDgYDVR0PAQH_BAQDAgWgMB0GA1UdDgQWBBQbwB3CifQh36XI9i1lBe8503G1RTAfBgNVHSMEGDAWgBSs43L6A7Jznj2VyO-HW67dG6HtaDCCAbIGA1UdHwSCAakwggGlMGmgZ6BlhmNodHRwOi8vcHJpbWFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwa6BpoGeGZWh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3R1czIvY3Jscy9jY21ld2VzdHVzMnBraS9jY21ld2VzdHVzMmljYTAxLzY3L2N1cnJlbnQuY3JsMFqgWKBWhlRodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwb6BtoGuGaWh0dHA6Ly9jY21ld2VzdHVzMnBraS53ZXN0dXMyLnBraS5jb3JlLndpbmRvd3MubmV0L2NlcnRpZmljYXRlQXV0aG9yaXRpZXMvY2NtZXdlc3R1czJpY2EwMS82Ny9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jYWNlcnRzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWV3ZXN0dXMycGtpLndlc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQACHqTwaTHdHWv9uHzVkq0ltJxzjLPIQHJnP6ndMrFnY-TIxQDrABl7eSPlINC2qk1CFExKLmjYM8aDst83BXP-yH0AHIvBrOGVdZvqFE-8KPpkqE97J5NNLNr1lLkz4vWpbjmJhLvYmureoMJjtcLpQyz-EfACVM90Lkmu8ueIkp5mjzxa_-cMa3KYwbFXLO8huXYcrhLhvs31Z9czJKjlPptL0ZE1-rBlMXGqTdZUE1CfPRoTjMMRZe_jEJYNKyxhL_5fT2mLxdlYJsMDeTU5qM-x3lA74zxVrUvC0ZvCQdK629aZQUbWV74Uzk1DmAtUiDhGvnsOW-rug_uLr621\u0026s=2ED070N0ZTuw-f1HVDJ70D4TL-E7a_pmemS26JAcsDWdtZNt5piI-6t7rp4DAeUvveysVwPzvAf5ULh-g26b96AAV9pXZ4I1vo74l2qjDBTA1ho2Ia11r_3U1XAtiPoLjGufya8CYXOek1iwZu7oaFEZ-4uayFIei-jUkQWxZ9Jz8_HKfyzJZZA1qJtJWKMIu_-fMQRELNXCWOkBgzgrUj1MP39GcY4oa5lZoVmPJGhHGseHZ--ETS4Vun62lwfxl7XrPb2PN0L9g7S07Nil12FIWSbh_qVJ2DC5m-7fPlkYze7LvocmWvHPwMOLiIbvvBS9Ui4BX7syQlC25zGKXw\u0026h=siGPFN1sMGFTEzhJalRHQiDtbRbqB8eZGXQRBJUWkyo\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qhh4b47m8ncv/providers/Microsoft.Compute/virtualMachines/cidr-connectivity-test-test-vm/runCommands/e2e-runcommand-1789704441845701237?api-version=2024-03-01\": stream error: stream ID 33; CANCEL; received from peer)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:293]: failed to create cluster role binding for external auth via VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d173e0\u003e: \n    VM run command \"e2e-runcommand-1789704441845701237\" on VM \"cidr-connectivity-test-test-vm\" failed and cleanup also failed: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/locations/CanadaCentral/operations/67651689-75ab-4f6c-aa6e-1746fbbdc2ad?p=c6bb2de4-260b-49a6-86a7-56c69bdc0403\u0026api-version=2024-03-01\u0026t=639253012427558463\u0026c=MIIHlDCCBnygAwIBAgIQTJW7e7XbMoQlWPa331WcnDANBgkqhkiG9w0BAQsFADA2MTQwMgYDVQQDEytDQ01FIEcxIFRMUyBSU0EgMjA0OCBTSEEyNTYgMjA0OSBXVVMyIENBIDAxMB4XDTI2MDgxMTE3MzMyMloXDTI3MDIwNjIzMzMyMlowQDE-MDwGA1UEAxM1YXN5bmNvcGVyYXRpb25zaWduaW5nY2VydGlmaWNhdGUubWFuYWdlbWVudC5henVyZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeQHh5tmZ6ZYRlf_Q_PwPJqUN9t1MEgnet-7jMlsIvkZBhjMDVPNEFS1UtIM-3StxxWCffcghG31P1UcexhHelJCujRWSZYWfEIC7z_S8BBwxDhcc2peEIdNKD1O6UKvglkD8KRsPxRT-ZwQYY3qFNngixnx3EDhtWr1a9Kg46bdJJAIIzu3flYJzQzP3yVrFuRQn4iH75UvhgEByfjJaD75Ugw9JYkgO52nE5dVc2ddxoRMWzJKxp_V2Adp7VqTbMx2T8hO9PWqkltIPwoAXIubXFj3Z5A6Wj7hE7XmY_2cFJKJfspjiqwLyfnLqlv9pCLhIVdG31g8DAGZM7fc2lAgMBAAGjggSSMIIEjjCBnQYDVR0gBIGVMIGSMAwGCisGAQQBgjd7AQEwZgYKKwYBBAGCN3sCAjBYMFYGCCsGAQUFBwICMEoeSAAzADMAZQAwADEAOQAyADEALQA0AGQANgA0AC0ANABmADgAYwAtAGEAMAA1ADUALQA1AGIAZABhAGYAZgBkADUAZQAzADMAZDAMBgorBgEEAYI3ewMCMAwGCisGAQQBgjd7BAIwDAYDVR0TAQH_BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDgYDVR0PAQH_BAQDAgWgMB0GA1UdDgQWBBQbwB3CifQh36XI9i1lBe8503G1RTAfBgNVHSMEGDAWgBSs43L6A7Jznj2VyO-HW67dG6HtaDCCAbIGA1UdHwSCAakwggGlMGmgZ6BlhmNodHRwOi8vcHJpbWFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwa6BpoGeGZWh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3R1czIvY3Jscy9jY21ld2VzdHVzMnBraS9jY21ld2VzdHVzMmljYTAxLzY3L2N1cnJlbnQuY3JsMFqgWKBWhlRodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwb6BtoGuGaWh0dHA6Ly9jY21ld2VzdHVzMnBraS53ZXN0dXMyLnBraS5jb3JlLndpbmRvd3MubmV0L2NlcnRpZmljYXRlQXV0aG9yaXRpZXMvY2NtZXdlc3R1czJpY2EwMS82Ny9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jYWNlcnRzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWV3ZXN0dXMycGtpLndlc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQACHqTwaTHdHWv9uHzVkq0ltJxzjLPIQHJnP6ndMrFnY-TIxQDrABl7eSPlINC2qk1CFExKLmjYM8aDst83BXP-yH0AHIvBrOGVdZvqFE-8KPpkqE97J5NNLNr1lLkz4vWpbjmJhLvYmureoMJjtcLpQyz-EfACVM90Lkmu8ueIkp5mjzxa_-cMa3KYwbFXLO8huXYcrhLhvs31Z9czJKjlPptL0ZE1-rBlMXGqTdZUE1CfPRoTjMMRZe_jEJYNKyxhL_5fT2mLxdlYJsMDeTU5qM-x3lA74zxVrUvC0ZvCQdK629aZQUbWV74Uzk1DmAtUiDhGvnsOW-rug_uLr621\u0026s=2ED070N0ZTuw-f1HVDJ70D4TL-E7a_pmemS26JAcsDWdtZNt5piI-6t7rp4DAeUvveysVwPzvAf5ULh-g26b96AAV9pXZ4I1vo74l2qjDBTA1ho2Ia11r_3U1XAtiPoLjGufya8CYXOek1iwZu7oaFEZ-4uayFIei-jUkQWxZ9Jz8_HKfyzJZZA1qJtJWKMIu_-fMQRELNXCWOkBgzgrUj1MP39GcY4oa5lZoVmPJGhHGseHZ--ETS4Vun62lwfxl7XrPb2PN0L9g7S07Nil12FIWSbh_qVJ2DC5m-7fPlkYze7LvocmWvHPwMOLiIbvvBS9Ui4BX7syQlC25zGKXw\u0026h=siGPFN1sMGFTEzhJalRHQiDtbRbqB8eZGXQRBJUWkyo\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qhh4b47m8ncv/providers/Microsoft.Compute/virtualMachines/cidr-connectivity-test-test-vm/runCommands/e2e-runcommand-1789704441845701237?api-version=2024-03-01\": stream error: stream ID 33; CANCEL; received from peer)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780897212043264","occurred_at":"2026-09-18T02:56:11Z","signature_id":"66f62a5d5d02e16dfffeec66474c962d8c14552e10473a54125784861158f0d5","pr_number":6898,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7855bddb074e5ee4da59762c697f802a687000bfd9477e9f92fc724e190b9330","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to get pod logs for pod fips-check-\u003cid\u003e in namespace e2e-fips-test-\u003cid\u003e on node \u003cnode\u003e after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv: failed to get pod logs for pod fips-check-rxfv5 in namespace e2e-fips-test-k5rlj on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-rxfv5)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc0014ee660\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv: failed to get pod logs for pod fips-check-rxfv5 in namespace e2e-fips-test-k5rlj on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-rxfv5)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc0014ee660\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv: failed to get pod logs for pod fips-check-rxfv5 in namespace e2e-fips-test-k5rlj on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-rxfv5)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569759195074560","occurred_at":"2026-09-17T12:57:11Z","signature_id":"b4cc553a2220034e450c979876c39e29fdc1278faa5ca56fae473d7a07ad2b7c","pr_number":6989,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f98a5615cab85c5989e6a0ac742f663ddff170e09cb4f716bf9a5978a279cf45","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to get pod logs for pod fips-check-psxxs in namespace e2e-fips-test-\u003cid\u003e on node \u003cnode\u003e after retries: an error on the server (\"unknown\") has prevented the request from succeeding","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5: failed to get pod logs for pod fips-check-psxxs in namespace e2e-fips-test-rx7hl on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-psxxs)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc002923158\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5: failed to get pod logs for pod fips-check-psxxs in namespace e2e-fips-test-rx7hl on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-psxxs)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc002923158\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5: failed to get pod logs for pod fips-check-psxxs in namespace e2e-fips-test-rx7hl on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-psxxs)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"64d1fa05ea54883aa2363d63f6b4eef64e006023913f4dea31d5e86c98373938","pr_number":6805,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc002e66048\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc002e66048\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","signature_id":"dcd0bfb37a59db9bd0b774c26b5a47b2e4002b0f9d6d7d9a6292d0c9dca75bea","pr_number":6966,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c4e2bd93d0410f48ddfa390cc77604492607a58b2a4eaff204baa6c7bed3af07","failure_pattern":"alert [svc] KubeContainerOOMKilled fired","search_query":"alert [svc] KubeContainerOOMKilled fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T13:42:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j4979456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.70:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4979456/providers/microsoft.monitor/accounts/services-j4979456\", namespace=\"arobit\", pod=\"arobit-forwarder-k26lv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9061149b-822a-4131-9e5f-b554ee1870de\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-k26lv on cluster ci01-j4979456-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T13:42:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j4979456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.70:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4979456/providers/microsoft.monitor/accounts/services-j4979456\", namespace=\"arobit\", pod=\"arobit-forwarder-k26lv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9061149b-822a-4131-9e5f-b554ee1870de\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-k26lv on cluster ci01-j4979456-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554708664979456","occurred_at":"2026-09-17T11:57:12Z","signature_id":"6db51492a3864e98312b98dbacda80d9a85398c3e3f34314664719d56a168f65","pr_number":6756,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a89f0d2edcd262580a6fd914503ced16ad52ae763f2c2255b4f5af55d375ecd3","failure_pattern":"alert [svc] KubeContainerWaiting fired","search_query":"alert [svc] KubeContainerWaiting fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerWaiting does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerWaiting does not fire","occurrences":1}],"full_error_samples":["alert fired 5 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:30:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"secrets-store\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-cwxxr\", region=\"canadacentral\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-cwxxr in namespace kube-system on container secrets-store has been in waiting state for longer than 1 hour.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:30:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"liveness-probe\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-cwxxr\", region=\"canadacentral\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-cwxxr in namespace kube-system on container liveness-probe has been in waiting state for longer than 1 hour.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:30:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"node-driver-registrar\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-cwxxr\", region=\"canadacentral\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-cwxxr in namespace kube-system on container node-driver-registrar has been in waiting state for longer than 1 hour.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:30:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", namespace=\"kube-system\", pod=\"ama-metrics-node-xfmjp\", region=\"canadacentral\", severity=\"warning\"\n  Description: pod/ama-metrics-node-xfmjp in namespace kube-system on container prometheus-collector has been in waiting state for longer than 1 hour.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:30:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"addon-token-adapter\", namespace=\"kube-system\", pod=\"ama-metrics-node-xfmjp\", region=\"canadacentral\", severity=\"warning\"\n  Description: pod/ama-metrics-node-xfmjp in namespace kube-system on container addon-token-adapter has been in waiting state for longer than 1 hour."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"97461ac292fa9fba61fa81d505ebf8a88d3f96ffd9a07f7d644245f3dd5aa26b","pr_number":6982,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T02:39:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T02:40:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T02:46:35Z\n  Ended:       2026-09-17T02:54:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-17T02:46:35Z\n  Ended:       2026-09-17T02:59:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7/klusterlet-addon-workmgr is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"4d675a2aa4e866465ba35a460544e188e20719f62123eb05ebda79f62db3498b","pr_number":6940,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d6c6b690ca60846dd98e464313d6a9672b9abd8744604edd915627eb68203f9d","failure_pattern":"alert [svc] KubeNodeNotReady fired","search_query":"alert [svc] KubeNodeNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeNotReady does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeNotReady does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:00:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"prometheus\", node=\"aks-system-28181016-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-system-28181016-vmss000000 has been unready for more than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:00:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"prometheus\", node=\"aks-system-28181016-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-system-28181016-vmss000000 has been unready for more than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"e58f291d243075248b87935d2709ed85f1b36fdb31f78f88aaae185cc551feae","pr_number":6982,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T21:40:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j0025600-mgmt-2\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100689397040025600","occurred_at":"2026-09-17T20:52:35Z","signature_id":"c8d2c94c46ea41bb6616321fc23a840d9b2ec094985319abefcb6af50ae87597","pr_number":6945,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d89bec3ea75e80fed983a698bbef1fbb78c33e77e634e6400cab6b214e710fe5","failure_pattern":"alert [svc] userJourneyFrontendLatencyP991h5m fired","search_query":"alert [svc] userJourneyFrontendLatencyP991h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP991h5m does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP991h5m does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T09:39:29Z\n  Ended:       2026-09-16T09:50:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatencyP991h5m\", cluster=\"ci01-j3290112-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3290112/providers/microsoft.monitor/accounts/services-j3290112\", severity=\"3\", short_window=\"5m\", slo=\"frontend-latency\"\n  Description: Frontend p99 latency on cluster ci01-j3290112-svc is above 5s on both the 1h and 5m windows (severe miss of the p99 \u003c 1s ARM sync SLO).\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-16T10:10:33Z\n  Ended:       2026-09-16T10:17:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatencyP991h5m\", cluster=\"ci01-j3290112-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3290112/providers/microsoft.monitor/accounts/services-j3290112\", severity=\"3\", short_window=\"5m\", slo=\"frontend-latency\"\n  Description: Frontend p99 latency on cluster ci01-j3290112-svc is above 5s on both the 1h and 5m windows (severe miss of the p99 \u003c 1s ARM sync SLO)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132038643290112","occurred_at":"2026-09-16T07:57:51Z","signature_id":"13f9b4b06b1b9850ff1976e03ab24843f0b045626fd218be7150c7dace6ffbef","pr_number":6503,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e9ea5f54afe3e98f1a64398661238c6fc0e342dcd1119b0d656cfccf60db80ec","failure_pattern":"breakglass session kubeconfig not ready: timeout waiting for session to become ready; HostedControlPlane exists but is not ready","search_query":"failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-wlfqkcdjh9rm/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-lbdxk/kubeconfig: timeout waiting for session to become ready (last status: {\"HostedControlPlaneAvailable\":{\"message\":\"HostedControlPlane exists but is not ready\",\"reason\":\"HostedControlPlaneNotReady\",\"status\":\"False\"},\"status\":\"Session is not ready\"})","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:185]: failed to create SRE breakglass credentials with aro-sre-pso permissions\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a7f820\u003e: \n    failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-wlfqkcdjh9rm/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-lbdxk/kubeconfig: timeout waiting for session to become ready (last status: {\"HostedControlPlaneAvailable\":{\"message\":\"HostedControlPlane exists but is not ready\",\"reason\":\"HostedControlPlaneNotReady\",\"status\":\"False\"},\"status\":\"Session is not ready\"})\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:185]: failed to create SRE breakglass credentials with aro-sre-pso permissions\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a7f820\u003e: \n    failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-wlfqkcdjh9rm/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-lbdxk/kubeconfig: timeout waiting for session to become ready (last status: {\"HostedControlPlaneAvailable\":{\"message\":\"HostedControlPlane exists but is not ready\",\"reason\":\"HostedControlPlaneNotReady\",\"status\":\"False\"},\"status\":\"Session is not ready\"})\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100153419804184576","occurred_at":"2026-09-16T09:22:48Z","signature_id":"6c4e73f54e8d63150bdff29f74282ba9a9e2ff1e60f7bd58d45e6cacc3dd7869","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fac5ab822da5a4db89037217494583b4069bba01be61d8f3a5122bc721b483e9","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"0IciLMVmLbqimBoU8f5q6QBEpmWjuJNgDzWaCM2v9MM=\"","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"0IciLMVmLbqimBoU8f5q6QBEpmWjuJNgDzWaCM2v9MM=\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001ffe8a0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"0IciLMVmLbqimBoU8f5q6QBEpmWjuJNgDzWaCM2v9MM=\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001ffe8a0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"0IciLMVmLbqimBoU8f5q6QBEpmWjuJNgDzWaCM2v9MM=\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252538870697984","occurred_at":"2026-09-16T15:56:40Z","signature_id":"b7562eb1317f094fe7e5e1b3fd0e3859a05304435b1ff984fb6d716f402b4c7c","pr_number":6791,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1ffc144947159d20ed7dd5010cb0a93b810641281a5d037d76b1920de5d6d537","failure_pattern":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later","search_query":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000b28b70\u003e: \n    clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000b28b70\u003e: \n    clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100747284332417024","occurred_at":"2026-09-18T00:42:37Z","signature_id":"82fd789c0354677fc94a578c2ebbcf48d32a60e2464d95dc9709afd5a7216752","pr_number":6972,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d33373d20e5996675bec12509adcc946f8d57b13e0f18cc57679d39e08115eb4","failure_pattern":"failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"\u003cresource-group\u003e\": Get \"\u003curl\u003e\": stream error: stream ID 13; CANCEL; received from peer","search_query":"failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8?api-version=2021-04-01\": stream error: stream ID 13; CANCEL; received from peer","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d270f8\u003e: \n    failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8?api-version=2021-04-01\": stream error: stream ID 13; CANCEL; received from peer\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d270f8\u003e: \n    failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8?api-version=2021-04-01\": stream error: stream ID 13; CANCEL; received from peer\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569681449455616","occurred_at":"2026-09-17T12:56:50Z","signature_id":"2126196a18f3d500d8a52360e541d2fceb67e8a67c46b2636cf54189a72bfe35","pr_number":6157,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e86119606257db722a985501dec6db3353e9294ec584e08a2cf350342957ce78","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context canceled]","search_query":"failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: context canceled]","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n  \"name\": \"c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-18T03:56:03.231032121Z\",\n  \"endTime\": \"2026-09-18T03:56:41.678977419Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: context canceled]\nExpected\n    \u003c[]error | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n  \"name\": \"c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-18T03:56:03.231032121Z\",\n  \"endTime\": \"2026-09-18T03:56:41.678977419Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: context canceled]\nExpected\n    \u003c[]error | len:2, cap:2\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","signature_id":"de20ef80e43125ed23aee61f6adcfb8942fa81561feaefb4e80cef19ed4db1c7","pr_number":6972,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"eaacc54ebd46cefe15abd433d5c3a394c2c9eab33016bf077e0076e05e8a3f60","failure_pattern":"failed to create route: the server is currently unable to handle the request","search_query":"failed to create route: the server is currently unable to handle the request","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-v458r6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001029700\u003e: \n    failed to create route: the server is currently unable to handle the request\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-v458r6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001029700\u003e: \n    failed to create route: the server is currently unable to handle the request\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023746776109056","occurred_at":"2026-09-18T19:01:10Z","signature_id":"c0b449ac08413f5f7083190089ef0d839099fa2699d64d1c9f2f8d26abbef529","pr_number":7010,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"40d94d1862472748004cc33c4c935e1c5142aa767d3a1a077dd2606d1dc7c7d3","failure_pattern":"failed to roll out Helm release: Internal error occurred: failed calling webhook \"validation.istio.io\": failed to call webhook: Post \"\u003curl\u003e\": EOF","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","occurrences":1}],"full_error_samples":["time=2026-09-17T09:16:31.171Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-17T09:16:31.636Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:53204456-a0a3-4fd2-89f7-36eb569bbb5d cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-ci01-j2597888] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j2597888.hcpsvc.osadev.cloud] image:map[digest:sha256:dfb4a580c4ae075df76e82458246c28e24407e254922d6c888d0d2793d3c1737 registry:arohcpsvcdev.azurecr.io repository:ci-op-6i9x21qi/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-6i9x21qi/pipeline:pull workloadIdentityClientId:0ee3f5a0-d80b-4385-abcc-91385fcf6f70 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:15494d5f-cf01-4a47-ae22-344126602800 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-09-17T09:16:31.636Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-17T09:16:31.786Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-09-17T09:16:31.977Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-09-17T09:16:31.977Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-17T09:16:32.031Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-17T09:16:37.667Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\"\ntime=2026-09-17T09:16:37.667Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100504522802597888","occurred_at":"2026-09-17T08:37:58Z","signature_id":"bbb80a3447e86b156cad4ed50a4810d3fa341869a205ecefaab92d1964bbcc6f","pr_number":6990,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e45f8e323244cd43106961bb8120aab24b3a867987bf8529ce7b7b4509a092cc","failure_pattern":"failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/18 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/18\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-16T09:03:26.822Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 description=\"Step kubelet-ds\\n Kind: Helm\\n\"\ntime=2026-09-16T09:03:27.157Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 values=\"map[enabled:true kubeletParametersImage:mcr.microsoft.com/aks/command/runtime@sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23]\"\ntime=2026-09-16T09:03:27.157Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-09-16T09:03:27.284Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 namespace=kubelet-ds\ntime=2026-09-16T09:03:27.475Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 dryRun=false\ntime=2026-09-16T09:03:27.475Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-09-16T09:03:27.530Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-09-16T09:08:28.583Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/18\\ncontext deadline exceeded\"\ntime=2026-09-16T09:08:28.583Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/18\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132003708932096","occurred_at":"2026-09-16T07:57:42Z","signature_id":"4592cebb946834bf2f5597cc8f68b2f39cdbe688bce563e773bde0e8a997725a","pr_number":6009,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1a0fe5d7569a83b55b15427e5bd80f1ae31a9f5785e0f4d3715f223065550ec9","failure_pattern":"failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","occurrences":1}],"full_error_samples":["time=2026-09-18T21:55:47.666Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-18T21:55:48.312Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[armPermissionsManagerIdentity:map[certName:armHelperPoolCert-6 clientId:9c78a811-9c85-4eb0-9c84-d0843866c888 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpci01oidcj1086464.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpci01oidcj1086464.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] backupCadenceProfile:testing backupScheduleState:Enabled clusterScopedIdentitiesConfig:map[roleSetName:dev] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:780fc8ab-b71d-4dc8-a753-5e608208a59f databaseName:arohcpci01-rp-j1086464 databaseUrl:https://arohcpci01-rp-j1086464.documents.azure.com:443/ kubeApplierCosmosContainerName:Manifests-MC-1 location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-vh2hkbkg/pipeline@sha256:ec4f5311c601606f0bdcb4316ea9dff923b912c8e13a93addd5ac2c23dd9e86c replicas:2 resources:map[limits:map[memory:unlimited] requests:map[cpu:475m memory:1Gi]] zoneCount:3] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] logVerbosity:4 maestro:map[sourceEnvironmentIdentifier:arohcpci01] miMock:map[certName:msiMockPoolCert-12 clientId:50d052a6-7aec-48b9-b43e-b8692a8f0ead clientIdSecretName: principalId:a76148d4-cb11-4b93-9363-54ba8239b0b1 principalIdSecretName: tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-vh2hkbkg/pipeline:pull workloadIdentityClientId:09708096-ca85-4992-8eb5-667a9b9d3fce workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:780fc8ab-b71d-4dc8-a753-5e608208a59f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-09-18T21:55:48.312Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-18T21:55:48.698Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-09-18T21:55:48.964Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-09-18T21:55:48.964Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-18T21:55:49.102Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-18T22:00:55.361Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"\ntime=2026-09-18T22:00:55.361Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101059271281086464","occurred_at":"2026-09-18T21:22:20Z","signature_id":"b8972dd0ff9a3ca0c20207306d152fa4afd8e4eb6a37b8d1c512cde87ef5ae3e","pr_number":6577,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"dac8a1b25a3188542fe843437616ad526549332051ce7885378b1b60aa94182a","failure_pattern":"failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kube-events (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kube-events (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-09-18T20:01:01.932Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 description=\"Step kube-events\\n Kind: Helm\\n\"\ntime=2026-09-18T20:01:02.379Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 values=\"map[enabled:true image:map[digest:sha256:5177425b194b5d289664c62bedbdc95a42605bbfcde48fc7ddbb31bee0194b90 registry:kubernetesshared.azurecr.io repository:shared/kube-events] resources:map[limits:map[memory:unlimited] requests:map[cpu:200m memory:512Mi]]]\"\ntime=2026-09-18T20:01:02.379Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1\ntime=2026-09-18T20:01:02.574Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 namespace=monitoring\ntime=2026-09-18T20:01:02.761Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 dryRun=false\ntime=2026-09-18T20:01:02.761Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1\ntime=2026-09-18T20:01:02.827Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1\ntime=2026-09-18T20:06:05.062Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 err=\"resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-09-18T20:06:05.062Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023754950807552","occurred_at":"2026-09-18T19:01:12Z","signature_id":"30e287a059c7694cef227767592b23353845f4281b98b8c7ff98febc06270f17","pr_number":7002,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"35fcc68c27835aae14daafdef12fc813fb2e38cc3e35d2f974eb6cc333b3ce7e","failure_pattern":"kube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane","search_query":"kube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:85]: Timed out after 900.000s.\nkube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:83 with:\nexpected kube_node_info metrics for at least one hostedcontrolplane but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:85]: Timed out after 900.000s.\nkube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:83 with:\nexpected kube_node_info metrics for at least one hostedcontrolplane but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100556682735128576","occurred_at":"2026-09-17T12:05:14Z","signature_id":"7be65470254fc4c585209a09735c416c75334c6ead21b5d5328120538ae66b13","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cea54d0df4cc4df84a4bbaf41795235750a198ce743843fbdff1373606efa1aa","failure_pattern":"kube_node_info metrics never appeared in Azure Monitor for cluster \"\"","search_query":"kube_node_info metrics never appeared in Azure Monitor for cluster \"\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:97]: Timed out after 900.001s.\nkube_node_info metrics never appeared in Azure Monitor for cluster \"\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:95 with:\nexpected kube_node_info metrics for cluster \"\" but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:97]: Timed out after 900.001s.\nkube_node_info metrics never appeared in Azure Monitor for cluster \"\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:95 with:\nexpected kube_node_info metrics for cluster \"\" but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"4645b1f607eafec38a4f58b597cf9ee778993ba4ba3bc944eda97fc9f0f70da7","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"662e49bfb814d1960ee5fb9e525e6ffab8495cd359fc51176893943560eee29a","failure_pattern":"node pool \"\u003cnodepool\u003e\" should have exactly one active version after initial creation","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:144]: node pool \"np-1\" should have exactly one active version after initial creation","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:144]: node pool \"np-1\" should have exactly one active version after initial creation\nExpected\n    \u003c[]*armredhatopenshifthcp.NodePoolActiveVersion | len:0, cap:0\u003e: ...\nto have length 1\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:144]: node pool \"np-1\" should have exactly one active version after initial creation\nExpected\n    \u003c[]*armredhatopenshifthcp.NodePoolActiveVersion | len:0, cap:0\u003e: ...\nto have length 1"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222891458039808","occurred_at":"2026-09-16T13:58:52Z","signature_id":"4c1a786ccdf5f4a6714330ab65d6eaf1faaac9d21fed54613d33cde0056d6e52","pr_number":6952,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"62df4554fa8988de83bcea5588b7f5f69715e1db3483f50b3137f97138b59e77","failure_pattern":"run command \"e2e-runcommand-1789636815233707722\" failed with exit code 28 on VM \"\u003cvm\u003e\" (output: \"000\")","search_query":"run command \"e2e-runcommand-1789636815233707722\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.136986301369863,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 654.956s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc00277e080\u003e: \n    run command \"e2e-runcommand-1789636815233707722\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 654.956s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc00277e080\u003e: \n    run command \"e2e-runcommand-1789636815233707722\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494074514509824","occurred_at":"2026-09-17T07:56:27Z","signature_id":"1395a5c67402b5879cd22d694e49480a23894641a58ba386a295937a83eff76f","pr_number":6752,"after_last_push_of_merged_pr":true}]}]},{"environment":"int","summary":{"total_runs":28,"failed_runs":11,"raw_occurrences":12,"matched_occurrences":3,"runs_affected":2},"rows":[{"environment":"int","failure_pattern_id":"a52d3f63256add31d93e084d23258696e3ce9f667510ec06dc509046b07f2f14","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD","STG"],"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000df6cd8\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-pfttlc9b26s6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\",\n      \"name\": \"ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T22:57:32.075098968Z\",\n      \"endTime\": \"2026-09-16T23:16:32.974740274Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000df6cd8\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-pfttlc9b26s6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\",\n      \"name\": \"ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T22:57:32.075098968Z\",\n      \"endTime\": \"2026-09-16T23:16:32.974740274Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100351631508901888","occurred_at":"2026-09-16T22:30:19Z","signature_id":"f7210bc017800c8d3ce9292e2b300557e56751eedb6338b78091f529330d0435","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"44813a24efc9a4e87de23a1d03f218de3f44f990811139bedf70528921d1e908","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:162]: failed to verify HCP cluster ephemeral-disk is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017886a8\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:162]: failed to verify HCP cluster ephemeral-disk is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017886a8\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100532556209328128","occurred_at":"2026-09-17T10:29:22Z","signature_id":"a7b733023a299f40faa1bcb49b8265ed28d3f85ddfbf335757885b57a1e7d20f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"221cb0b46f3bef91cff2ee06c068eca8fb3646ef7eeb7ed9cd39ae59c1446e81","failure_pattern":"list nodes (nodePool=\u003cnodepool\u003e): Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"list nodes (nodePool=npdg-4-21): Get \"https://api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net:443/api/v1/nodes\": dial tcp: lookup api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:59019-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:612]: failed to get node release images before downgrade\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0019a0220\u003e: \n    list nodes (nodePool=npdg-4-21): Get \"https://api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net:443/api/v1/nodes\": dial tcp: lookup api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:59019-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:612]: failed to get node release images before downgrade\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0019a0220\u003e: \n    list nodes (nodePool=npdg-4-21): Get \"https://api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net:443/api/v1/nodes\": dial tcp: lookup api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:59019-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100532556209328128","occurred_at":"2026-09-17T10:29:22Z","signature_id":"06051e7fd69c31d191475d70ad084a2f5f338204d9dcb2ccb5a0566c0fc34ac7","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"prod","summary":{"total_runs":35,"failed_runs":29,"raw_occurrences":118,"matched_occurrences":103,"runs_affected":14},"rows":[{"environment":"prod","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":51,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":51,"after_last_push_count":51,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,0,51],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-n6q68g with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000be8cc0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-n6q68g: failed waiting for cluster=\"np-version-upgrade-cluster-n6q68g\" in resourcegroup=\"rg-np-version-upgrade-n6q68g-z5wvsgjdr7l9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n      \"name\": \"47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T19:55:24.695867176Z\",\n      \"endTime\": \"2026-09-18T20:24:27.02816156Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-n6q68g with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000be8cc0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-n6q68g: failed waiting for cluster=\"np-version-upgrade-cluster-n6q68g\" in resourcegroup=\"rg-np-version-upgrade-n6q68g-z5wvsgjdr7l9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n      \"name\": \"47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T19:55:24.695867176Z\",\n      \"endTime\": \"2026-09-18T20:24:27.02816156Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de50e0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-bddwvtnwz8x8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n      \"name\": \"1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T19:54:45.082839017Z\",\n      \"endTime\": \"2026-09-18T20:23:47.024709747Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de50e0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-bddwvtnwz8x8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n      \"name\": \"1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T19:54:45.082839017Z\",\n      \"endTime\": \"2026-09-18T20:23:47.024709747Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010085a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-266qhn49mfp5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n      \"name\": \"724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T18:06:21.25552087Z\",\n      \"endTime\": \"2026-09-18T18:35:23.923631594Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010085a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-266qhn49mfp5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n      \"name\": \"724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T18:06:21.25552087Z\",\n      \"endTime\": \"2026-09-18T18:35:23.923631594Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster placement did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"020177279fd83a306292e61e3b6eb559ce1bd5ba1e079600d6fe28da054a781b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"08737fbd1b615be947880829b89bc4c3a56995f666d4512eff62626a308dd556","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"087e1145946b4175b23e226d2c422f8fbe6adcface041612342c65176f135291","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"12c88ab455c66941eb056387e08ab482c4399c448c1844b1db1d64ca2a21923b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"18459d5e026c91320fea1e6f4619d65e0f3c2cce677a7bf1890dbe9529aa82dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"18cde222ab56ffb759e053d76ee6e19c739c516c2db2d311138d345c27569034","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"19596332cc8edb7c2fe7ff05b62421eeeea53668cb594d6f172e695930cfc93e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"23eff0283d14160643a2c1b951b106f62c524eb771d25036e9a90406a0924c53","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"279014bba3c91c8c1147706aa208a7ca8dab431b40a05ae4dc2507ffe4bd82d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"2a4e0071788107d1b0f28cafcae777b74d59c2ed92ca22e3caeb7564b1685be1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"2bd526e04864e297297108f31c386e629877109b67a0cd5577f66e966f4820a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"2c5047fbc800377676e3159b01faa4b14e97abaaef72bb1d0f7c177a6a8e963a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"2e5039616cb1c8d7f4ac14c50b2a06f1fe40081131c5acd543f28d2cc514abec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"300283533309312abfaf8eb06c5bc243249dba952ac99795631a8cca96c1b04d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"3a36b2cb0e45db6b1b5adf537bf6a65a0dc2daa7ebc74fcf104789a0636dc715","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"4d196dbdd6adc5cbd0b42174989b35c61699ec802a9b43e61050ca43a4a526db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"55df98f2e805e8d535825529f230ec77e5caf64b117f366392092a2ef0f04238","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"55f11975675616706103b6b7daff6a4b423e0747ce59c5e7da66f3dc349981e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"596b0ffa1cf24f21e4c89da0dd1bf8032daa39c9dfe890cbc997391f5e16d758","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"5bd085aa7d4e7efc364c83356bec393db7595ff945fba164d18f342fedc7d074","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"612c1b0bfa8c63e918534f55b46917bda5292bb668f43f5109397135c5f16839","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"7d0f6a2cda1c21acb4480d38818269f535fbffb2494dac6f98b849200db6bc5d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"807a30302c1afe55628a0e91c1d8d2d1cf846d02300aeafd38841fdc5629959a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"88f393269cef98e4be13d3c8ab69c7daf8948bb76a2277ef306df923eea104ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"8c17a49c3b1087a5252885b16d63b8887ba96a24c838b70dae74c83a59595562","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"8e3503c7dac56f3fd400f32d8b3f5b71ecf5acd0c55bc2d5e52b0b130348d093","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"8f8a4a3e3e075b660c1fb0534694eabcbec0ee97ddf1c7de50ae027a7c989e5e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"9253c3493335b5dfd398622bbe3451ad8bddf74417d1a8c0037aece178838cde","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"980439aafe10d6730454288b03e5c4b325ad4d9a23209ccce9ea0925ca8727b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"99a0fc3be3299d125a1312f83223671d90940dffaa62fee2e5b18166cfca67f4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"9da2d69669521624ba370e29cdccd478000fd77fa5acf531ed0100da09e598f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"a086d8d16ebbda1f55085340c84766813ece581729f4e4db8f61f9c0b1339c8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"aba4ff8b5edea60411faee257f0fcca1514022f58c987167a980974659ac599f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"ac027a2fae7f9c119c5d13e433aca363beeb537c7f13236bce5b4509168d72e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"b09c9c83b49154876ea9edaee019cac3ced0de0e27ddbe6889ee50f6b423f2ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"b971663cd5f630a57997cfad6d7efdb8293b9c2c40b6d6ab7621acde7a5c5511","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"bec4f85eb91db9f106f1504bbcfbd5d619f318abcff9dccca5f00cd3af48e378","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"bfbf541997c9dcf2457e39f6d9a3f3675b64df774c7c81e642e24bee8b14665a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"c71f8c34107674e9b83007fd4c4382acbf933644d3d70e04b10c2bce3d213437","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"c9577e8c1b375d41d27d0e18df18642cc9e2a302455c27112af6cdf04b27da87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"cb28d0b51212f011a6a9a6a3530556dc58c4a4d691d64d362e422cfc091e42ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"ccebff601285f1ea3ac04ae7a355282a8c0e14e4f8eb582eebc840c11f6d9a08","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"cf464e2390b41d88251c211ae76f31739afd822d6d94414c2b9fbfc2f709feaa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"d992a344dec95549926180b213136a61987bad3b2b2cfe5a8cedabfb0cba64d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"d9b87f83858caf3b228ef9b132bcd625c28601b6944272225765d7e4212bfbd4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"eabc0e9f060f2e7a3e41ea442762aca89d3441bef1655db8dca9669defa6ff8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"ec2f8559597c2e2e96353159ad3c18cb0e7f4c1ae2d3f60c226c0178a7ef5d2b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"ed3aaafec71e925920ce96600e6accd52368d8c067bd7c98fb0f3dd54c8990b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"f52d9598c009df2683fa8e80fa13ffac13c236506f242fa3de8742e288cc9518","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"f6a9a5ad677f486ebe2e2116b48d59eff2dab4dc02f9785191be923be7559549","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"fcaef2f74196ab4f30c8d92bee5653f6add69a944685179ec76976f1aebee54b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":14,"runs_affected":3,"failed_runs":3,"run_impact_percent":8.571428571428571,"anchor_occurrences":14,"after_last_push_count":14,"also_in":["STG"],"trend_counts":[0,0,0,0,0,5,4,5],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f5e360\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-cx8sqpwt9s45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f5e360\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-cx8sqpwt9s45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002bb68a0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-t78vdzzqpblp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002bb68a0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-t78vdzzqpblp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002804f90\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-grcsb5ww9xqw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002804f90\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-grcsb5ww9xqw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4e7909db88252003612c7eb675142841e8737860b4601bc7ba1c1c1147d59585","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"6b756bbe3882dd3ad1f64906dbb7701108fcd8092065edc0fdfc339e155785c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"9ef798cef3d572a0e66143f21817eeeceae70fa379c7c12f948b17be196688fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"a1f4334212b041c83192bb8549659dd10bbe7f9e8bd239e3cb44f42460834cd6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"d656f1256e1df9e299a9ee99092b293d1d1b5f5763759abb016ae0c94e36cbee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"19d9edd34f8e48701cd8bbda924191e80690cc1893c6e47aa7458abc045a4095","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"ad4219e21e3f54d13de0b5a81134597e91fe66a1442cf0f6968126d652406ccf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"b64e2e209238a4291f7bc6ca875c4002b2b72ce61c22b00cc341eff1d3d7d329","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"d144fcda6ced4631fcc19c7abd504ffe793c4fffbba643dd7b8a0d80f55c67e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"494d690c10401d515d7d97bb1b43282cd5512bedf4a8cf0e93991a312d0f71e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"5f0d2310ed06414a49e9f1ddd637c2a2516af067a00a8168124b28d160b1f9b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"a91c26d043927cb825f419af86a21ed7c54daa9e6da09ecdbe0fab7af0a8ab85","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"b2ee8d4e7b429ad69336ec70ed0fafeb7a58d728ccf645b31d75121fd78def37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"e2f2d3555780f9a4be9e4c49fe522f13754292efbc3c1299543a5305777d3351","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":11,"runs_affected":7,"failed_runs":7,"run_impact_percent":20,"anchor_occurrences":11,"after_last_push_count":11,"also_in":["DEV","INT","STG"],"trend_counts":[0,0,0,0,0,1,5,5],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002d105c0\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-wbftvl6pxn6v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f5be950-3d76-400d-bba8-80953e9b85fb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f5be950-3d76-400d-bba8-80953e9b85fb\",\n      \"name\": \"9f5be950-3d76-400d-bba8-80953e9b85fb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T07:41:11.725184381Z\",\n      \"endTime\": \"2026-09-18T08:00:14.656645268Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002d105c0\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-wbftvl6pxn6v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f5be950-3d76-400d-bba8-80953e9b85fb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f5be950-3d76-400d-bba8-80953e9b85fb\",\n      \"name\": \"9f5be950-3d76-400d-bba8-80953e9b85fb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T07:41:11.725184381Z\",\n      \"endTime\": \"2026-09-18T08:00:14.656645268Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.32\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000efdee0\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-dd2djv\" in resourcegroup=\"rg-np-version-upgrade-dd2djv-f5pt9zl6csx7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fcd870-f035-4d43-a07e-056005a0cd12\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fcd870-f035-4d43-a07e-056005a0cd12\",\n      \"name\": \"34fcd870-f035-4d43-a07e-056005a0cd12\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T03:23:36.136458591Z\",\n      \"endTime\": \"2026-09-18T03:42:38.777220989Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.32\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000efdee0\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-dd2djv\" in resourcegroup=\"rg-np-version-upgrade-dd2djv-f5pt9zl6csx7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fcd870-f035-4d43-a07e-056005a0cd12\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fcd870-f035-4d43-a07e-056005a0cd12\",\n      \"name\": \"34fcd870-f035-4d43-a07e-056005a0cd12\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T03:23:36.136458591Z\",\n      \"endTime\": \"2026-09-18T03:42:38.777220989Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001092648\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-tgqlbhrjcjzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4e2d962-cb10-4c5d-b5e9-88c9efab7468\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4e2d962-cb10-4c5d-b5e9-88c9efab7468\",\n      \"name\": \"f4e2d962-cb10-4c5d-b5e9-88c9efab7468\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T02:15:45.572313063Z\",\n      \"endTime\": \"2026-09-18T02:34:50.003063171Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001092648\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-tgqlbhrjcjzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4e2d962-cb10-4c5d-b5e9-88c9efab7468\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4e2d962-cb10-4c5d-b5e9-88c9efab7468\",\n      \"name\": \"f4e2d962-cb10-4c5d-b5e9-88c9efab7468\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T02:15:45.572313063Z\",\n      \"endTime\": \"2026-09-18T02:34:50.003063171Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100826864959361024","occurred_at":"2026-09-18T05:58:50Z","signature_id":"7cb7501ef4b819fdc17f5b5e3ca2619d486971f5a3c25a5533b8ff273a412172","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","signature_id":"60cc795ca9532d84d0a6a8c864d8b05a2f5285c1bfd6c8aa612a88d8890d8bb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","signature_id":"6e4745afece375c35c57231f904892210b15b74988a6f2baceda810fa43da8e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","signature_id":"8be7c23a1e56f277787ec21fa08eb46e46af4b8580d42385a8b71372839091d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100762866352132096","occurred_at":"2026-09-18T01:44:31Z","signature_id":"812ff7352462ae5b2ffc2a98177a772e72a7a199983bda3af2609cfd08e8cb77","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"8af91b6a912b93a55a51953f18f51b6e88bc18db738006d3b71d9e5d820486c4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100694048263114752","occurred_at":"2026-09-17T21:11:04Z","signature_id":"be41217a9374881e99f1a9416ccadf32d5a5da3fe931b3a6439f304be981164d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100694048263114752","occurred_at":"2026-09-17T21:11:04Z","signature_id":"ce783312029a03f3c6f81cf899cc96d5f27f953623a9a5fa1b6b579e78455f50","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100694048263114752","occurred_at":"2026-09-17T21:11:04Z","signature_id":"e9d197f55b7c8094b6f94fd2ac343ddfe7d54231e67ed0c0282b273041aea3b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100404440946184192","occurred_at":"2026-09-17T02:00:16Z","signature_id":"ce1e7db080deea06cb7610e1f4b4bf939ebd4afb01e8c1a7763c9d9cd1a9bd32","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"aae2457e0f58577b233ddb5159361da1324cc74ad01ffe504591fd6f9131c0dd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":8,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.714285714285714,"anchor_occurrences":8,"after_last_push_count":8,"also_in":["STG"],"trend_counts":[0,0,0,0,0,0,4,4],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00011be60\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-tcgt6v\" in resourcegroup=\"rg-np-skip-minor-tcgt6v-cwclsxtppk9g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-tcgt6v-cwclsxtppk9g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-tcgt6v/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00011be60\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-tcgt6v\" in resourcegroup=\"rg-np-skip-minor-tcgt6v-cwclsxtppk9g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-tcgt6v-cwclsxtppk9g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-tcgt6v/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013480c0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-mpcclt\" resourcegroup=\"rg-np-dg-minor-mpcclt-kpfflj2pg95l\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-mpcclt-kpfflj2pg95l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-mpcclt/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013480c0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-mpcclt\" resourcegroup=\"rg-np-dg-minor-mpcclt-kpfflj2pg95l\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-mpcclt-kpfflj2pg95l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-mpcclt/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00160a220\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ddwqfs\" in resourcegroup=\"rg-np-version-upgrade-ddwqfs-m7xcgsn6lcv6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ddwqfs-m7xcgsn6lcv6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ddwqfs/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00160a220\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ddwqfs\" in resourcegroup=\"rg-np-version-upgrade-ddwqfs-m7xcgsn6lcv6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ddwqfs-m7xcgsn6lcv6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ddwqfs/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"491628b312668f06a0a9bf2464e7de82cea05fd0fafcb43e52e269cbc904606b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"8807298fe1e8269d8ba1826e326fe8888ecc37b86d4680f87d9ef650960b1b45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"889f4b2cac9cd859f4a4017cf24ccfb20aa30ab6ad074847bdec286491246277","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"b07971657c34858950043966384bf3450b2bd8793b71995b336ae5e99ce5d756","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"475d18438d6129ee057302e14cb4929cac287683cc42b505124692ed9000396d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"5b8b71dd55429c7ea7e12ce3e8d437a79ae271abc6b3fd1cf66b2d542a2f54ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"9f76555451648e830890a7c9cd43251ffcd8ea938c9675715ab08891b03af525","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"ae583f4fcd37f743d2e4acb91167ea667e50e9c13e01db10c235ffe6dd99caa7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","failure_pattern":"ERROR CODE: SubscriptionNotRegistered; provider Microsoft.Insights","search_query":"ERROR CODE: SubscriptionNotRegistered","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":17.142857142857142,"anchor_occurrences":6,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,0,1,5],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a45890\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: SubscriptionNotRegistered\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"SubscriptionNotRegistered\",\n      \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a45890\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: SubscriptionNotRegistered\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"SubscriptionNotRegistered\",\n      \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00117a2a0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: SubscriptionNotRegistered\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"SubscriptionNotRegistered\",\n      \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00117a2a0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: SubscriptionNotRegistered\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"SubscriptionNotRegistered\",\n      \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00088e570\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: SubscriptionNotRegistered\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"SubscriptionNotRegistered\",\n      \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00088e570\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: SubscriptionNotRegistered\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"SubscriptionNotRegistered\",\n      \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100894117604102144","occurred_at":"2026-09-18T10:26:04Z","signature_id":"3269dcb6be86d6a6047ecae085ca193c45aec85e3774519113b1593c115dfa87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100892735677403136","occurred_at":"2026-09-18T10:20:35Z","signature_id":"d7f2aa1ec90b6da0508d97952c26f5e68a44fec6becea93e11a0c8acd2ecf80a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100890223054426112","occurred_at":"2026-09-18T10:10:35Z","signature_id":"5533b31e4276757a4afc74d1ad67148e45245ccef8ea260d65b9e08077693a1d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","signature_id":"6fd32498c414e944f0c010bd303fd297e25f438aa938164d7b89331e4fe1bf3e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100762866352132096","occurred_at":"2026-09-18T01:44:31Z","signature_id":"f164ce20210c5901433b68d8b32c7bafd9db9fbdb4b9f088ddfe4a53eb8ff36e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"a0e7d58faa2918f496570ed9c5f40cc4898c7cc2d361ef4097b193762f74fdcc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":8.571428571428571,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["STG"],"trend_counts":[0,0,0,0,0,1,1,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00298e7c0\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-9f9tbz46mct7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-9f9tbz46mct7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00298e7c0\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-9f9tbz46mct7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-9f9tbz46mct7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00079eb40\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-8mb2dp88brbx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-8mb2dp88brbx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00079eb40\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-8mb2dp88brbx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-8mb2dp88brbx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d85560\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-jzkmnxzs9c52\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jzkmnxzs9c52/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d85560\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-jzkmnxzs9c52\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jzkmnxzs9c52/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"19958c8a5897194ef3182c6ab16be4ac9029906aade55bb4256e8a9bbd189d3c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"eedd7ed28a2e498144c6499476b15f9b62b521f665937a7144186b5981a0ab86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"70af6c4c5a716b4a2eb8db0539e648de82b2496a872a99a8c6a98074fd41d4c3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":8.571428571428571,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["STG"],"trend_counts":[0,0,0,0,0,1,1,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00055c870\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00055c870\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.009s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000fe8ae0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.009s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000fe8ae0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00152ab10\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00152ab10\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"31f227f2999d078a72461d40e7b0f7d39a61c55e78688c72750e61796cea0c0e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"787555a75756b0bcd3c50e5022f58b6e6c990b64d6f87d9979ae97767e7916f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"f5ce501d2dd69dcdea5a25282314ce7ff4dede1a06ceeb918b7e36499f9665b6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"e3e491019ecf3f140e87c882b7d45ae9d3071ed1df24cc639d33c7a9d0e29087","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.714285714285714,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014ced60\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-wvt6tvvldjdg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n      \"name\": \"80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T18:06:15.268087105Z\",\n      \"endTime\": \"2026-09-18T18:35:15.027123676Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014ced60\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-wvt6tvvldjdg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n      \"name\": \"80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T18:06:15.268087105Z\",\n      \"endTime\": \"2026-09-18T18:35:15.027123676Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000860220\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9vjp5gmczk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n      \"name\": \"df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T02:08:07.344249043Z\",\n      \"endTime\": \"2026-09-18T02:37:10.01628355Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000860220\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9vjp5gmczk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n      \"name\": \"df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T02:08:07.344249043Z\",\n      \"endTime\": \"2026-09-18T02:37:10.01628355Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003701249380352","occurred_at":"2026-09-18T17:06:01Z","signature_id":"e22784c3104fac10937a42e7b5069677dd3f893046b62e795e223bfab93b9fcd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100762866352132096","occurred_at":"2026-09-18T01:44:31Z","signature_id":"736f2b0814fabfc59b7e462d9755a0e748e4d396c2743f81540491fd582dab16","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b57f3261aa83aadd9ceb7988b00830acc29894d12dc4f5f19baa862276967565","failure_pattern":"cluster operators not available: console (no Available condition), dns (Available=False, Progressing=True, Degraded=True, Upgradeable=True), image-registry (Available=False, Progressing=True, Degraded=True), ingress (Available=False, Progressing=True","search_query":"cluster operators not available: console (no Available condition), dns (Available=False, Progressing=True, Degraded=True, Upgradeable=True), image-registry (Available=False, Progressing=True, Degraded=True), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False), monitoring (no Available condition), openshift-samples (no Available condition), storage (Degraded=False, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown)","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc002a4e018\u003e: \n    context deadline exceeded\n    cluster operators not available: console (no Available condition), dns (Available=False, Progressing=True, Degraded=True, Upgradeable=True), image-registry (Available=False, Progressing=True, Degraded=True), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False), monitoring (no Available condition), openshift-samples (no Available condition), storage (Degraded=False, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc002a4e018\u003e: \n    context deadline exceeded\n    cluster operators not available: console (no Available condition), dns (Available=False, Progressing=True, Degraded=True, Upgradeable=True), image-registry (Available=False, Progressing=True, Degraded=True), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False), monitoring (no Available condition), openshift-samples (no Available condition), storage (Degraded=False, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100173746357669888","occurred_at":"2026-09-16T10:43:35Z","signature_id":"cfd34eb004cc5a7cf7577f79d571b368c1d8dfc53041aa652f9465c361a3b4a0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-tzdswnvvvrrb\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001376700\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-tzdswnvvvrrb\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001376700\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-tzdswnvvvrrb\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"e69d26f721f1d960b9d056d864133ef61ba83a27918de693c75cfc743d5fc4a8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ql4cwwsm948w\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-ql4cwwsm948w, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010a86c0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ql4cwwsm948w\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-ql4cwwsm948w, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010a86c0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ql4cwwsm948w\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-ql4cwwsm948w, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"60ea4497c6186b3e803d7e6ac044255c8c915fd98b1ff264e2058f0d46b0df0e","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":20,"failed_runs":19,"raw_occurrences":84,"matched_occurrences":70,"runs_affected":5},"rows":[{"environment":"stg","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":52,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":52,"after_last_push_count":52,"trend_counts":[0,0,0,0,0,0,0,52],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a102e0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-5t8m268g2ktr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-5t8m268g2ktr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a102e0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-5t8m268g2ktr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-5t8m268g2ktr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008ac220\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gp6ml4ltbxq9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-gp6ml4ltbxq9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008ac220\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gp6ml4ltbxq9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-gp6ml4ltbxq9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bba1c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xm5czf8ql9ll\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-xm5czf8ql9ll/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bba1c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xm5czf8ql9ll\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-xm5czf8ql9ll/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"085c6afa73df8e5dcbb8a9e0c59a48a53cb276afb92a752caf002a49c403af63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"0e7c33b837c87670cd35a11641aefa3118943334f501baf884bc7c01141848c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"0f82bd0c9700cf4a07c13d1f48415f495ffac7b49c2a7ae3841ea9c8faf79ca5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"10f37d08fe2d36b0048e252aedb2d548e4ca8db071149c417d5ce17b13fa36d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"1153df3f24dc8bcdeecfdd99c8141522fe1f0d9dd14f566bee678a30941ac49f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"1dd5e0dc7e15d356bc9a171360fd68f57e762e2e53e6cea924436fb68b6fb102","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"252c4afaeebb14c4115f0be666abd104e7adc80f40991bcc6beaa3ff85c91e84","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"25dbf965d6f20a1a7cec9776ec1e87245a3e50198ba091a7741329a88e0da378","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"265ce4c4fa80432d1672154f22571f629b7a60e7c351781f21d111b2d1880927","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"2740332bfcda380c3c02e8ee1f792b3568b04678a77023d1486f19de701aeb61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"27e6130cf36baa158d6711ebe28c73dffa820785f17bbc686ff8fc7d38fbdb8b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"340421cbdb7899927b7ac5fc7d0aa57d005e6eb5d784359ad31a36aed34461d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"3f18ddd10272127c408a7d9bde4eddaaa27e6f085f80cf7d94ab9be21a380754","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"42f7a4e6cab05c380f4d5d815367e49f29a7e4b22dcb36d855a4a05388b7e8cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4455b438d122ea70538db65aa4dcf60523e82a18567f93d3872e7891ddeb2d9f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"44de21e589e428af479c9ccfb6a72f778841ee48c0b271fb47903606d3baee24","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"483e21afcf9632ea6968f51c68da44c7e6f45d9a5418bb6f513b2bec6b1bbce2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4ba5ad9b16c83f433ab867cff2637f8bd36aa451460f9d15a478f5eb1eda9a65","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4c2e2bc46dbcb65a27b428b8f5c2170eb0f11ba205842dd52e99ae5bde4b31fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4d36c1670f69a8378c623f46ed5e24f3787ec0a5de39295ced2ebfe81f29cefa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4f0f437ca045765d2d2b24079491f21e1fd5a85db0b44958c8034b2e66199908","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"562b8f83f84dba1886b3065181ffee89aab2e6673b7333d41f76aef2eaad1f4e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"58f07f406d642a91c9bddbe8fdd7ede440f08a2dd03a6f5a5c819b498466cf11","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"5e002cf10865e514b0815eed29286f8da9237e5aa8330589afaeef395b289efb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"677da6ac04d265e57cafd47671de87d4257ef7d06221d84329ebe1ab461fca66","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"6adb067c216551bfe29f1e3b77d6113e9d3d513c52d7c6b6659f174eed5dbd71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"71d45e9348cce618a98ba3a9cedd12637edd7a5d0db05784f92b3cf08108bd98","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"71e62b511a0fe354af7045a804df32da837a8d8a1285d20120469dc2b9b474a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"8b5bf6763af34e2d91fc69a112089a080b1da4da152d93af2ebb7174e191d48d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"97623f5d3f0d0fd367349a611b76873782d9adb09766bc94fe24252159b1769d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"998b87f452d353eeb973a256989175c16ceae9088c3fe30ce72eb5beea463849","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"a4bd95380032fc5885c8b2fd10eb51f3df30c29355f6b414b5b33fef1056e9a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"a9d975f128c27c084e7e5fd23daa85c45a8d628e9193f529805c3f218b289677","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"b44a4a5abf9df504b29cd303adc2b536d491938b5012cb47d37469bc577abd3d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"b89878daa0c4d5d3cb3d93a46690c3d9ddf063236bf34cd2f69bcdf3bb5f245a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"bc25c842a1235045614d605acc90fea01e39dbaf7d68ff406505237e93f1e16d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"c0258f3c8ab984584c9b6314be2ccb30e5ffa2b206f4ab774c595c07d6b813f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"c3410d264cf0132fe4f17940cdf2ec821ec784b23b0faff95c971f7658965387","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"c68022f2f40c275eb8c23a59749c7f7e4af0d44ada9136f9192d94b24e86254b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"cb988d58c89113961f03df82043150415e9b6ed049bf1f878c0c0b5c9eb97486","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"ce5b35b01c844ca412a4cc92791058f44a790b6ff65cc19fb352ae11b15d2ae3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"cf3269165eebc8b281476a0026c2b3449c115912f9f0f7f746f7524bfed7b58f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"d66e9a51e6356e3da95cc4251424afb6ce526cd142562c82ca1f05f8b3157c1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"db766735f9e579f4552ed8a3c29b312be7add829d329efde1debee620ae19b46","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"e1a746b36f9a4f74e169ab5b044f8ffbb4e94a6ac47cce6553d8ffee67ae68fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"e5ac7068f8a4f91ce6b642a6a37bd31dd8ace822a1805ae8325512e343c782e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"e94453651233184d87c4057afb3b9acb271e36b21383edb286efa6f234f89bef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"f0973137271b8ecc421edadd409b7fe63accbff0882cef646035b79a0aa8c22e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"f0cc45ea9fca99b68f2c8c623264abfb0139dd3f1bc78462db9705e9cff98ef3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"f6f1275a16afdec341158ade8fbc4ce79c93972342a64ac61c74f29d86d94b33","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"f7953744450082a3ec5bc94fd5cc7b0b148b3213f668f2afb80a43e6f0953ba9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"ffc3ee48d637c5b8ce881fb48a931d89a1627b9e6bc94e2411754ee778248a76","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":5,"runs_affected":3,"failed_runs":3,"run_impact_percent":15,"anchor_occurrences":5,"after_last_push_count":5,"also_in":["DEV","INT","PROD"],"trend_counts":[0,0,0,0,0,1,3,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-h29zrk\" for cluster \"cluster-candidate-4-21-h29zrk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009df3c0\u003e: \n    failed to create NodePool np-h29zrk: failed waiting for nodepool=\"np-h29zrk\" for cluster \"cluster-candidate-4-21-h29zrk\" in resourcegroup=\"rg-candidate-4-21-k7gt4s25kh5h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eba02645-cd7e-4330-9e90-bb7a800d75d5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eba02645-cd7e-4330-9e90-bb7a800d75d5\",\n      \"name\": \"eba02645-cd7e-4330-9e90-bb7a800d75d5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T02:38:03.543368376Z\",\n      \"endTime\": \"2026-09-18T02:57:09.03735448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-h29zrk\" for cluster \"cluster-candidate-4-21-h29zrk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009df3c0\u003e: \n    failed to create NodePool np-h29zrk: failed waiting for nodepool=\"np-h29zrk\" for cluster \"cluster-candidate-4-21-h29zrk\" in resourcegroup=\"rg-candidate-4-21-k7gt4s25kh5h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eba02645-cd7e-4330-9e90-bb7a800d75d5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eba02645-cd7e-4330-9e90-bb7a800d75d5\",\n      \"name\": \"eba02645-cd7e-4330-9e90-bb7a800d75d5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T02:38:03.543368376Z\",\n      \"endTime\": \"2026-09-18T02:57:09.03735448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fc2640\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4xfvdn4xsffw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98b08bb9-0eee-4840-9535-91464d8458a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98b08bb9-0eee-4840-9535-91464d8458a2\",\n      \"name\": \"98b08bb9-0eee-4840-9535-91464d8458a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T05:40:34.02322608Z\",\n      \"endTime\": \"2026-09-17T05:59:42.272250066Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fc2640\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4xfvdn4xsffw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98b08bb9-0eee-4840-9535-91464d8458a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98b08bb9-0eee-4840-9535-91464d8458a2\",\n      \"name\": \"98b08bb9-0eee-4840-9535-91464d8458a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T05:40:34.02322608Z\",\n      \"endTime\": \"2026-09-17T05:59:42.272250066Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0035265a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-rcr2wwpjvcjv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b397553-ee61-4674-b26e-e3cf8ae2dc9d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b397553-ee61-4674-b26e-e3cf8ae2dc9d\",\n      \"name\": \"5b397553-ee61-4674-b26e-e3cf8ae2dc9d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T05:39:36.012772677Z\",\n      \"endTime\": \"2026-09-17T05:58:42.168749593Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0035265a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-rcr2wwpjvcjv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b397553-ee61-4674-b26e-e3cf8ae2dc9d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b397553-ee61-4674-b26e-e3cf8ae2dc9d\",\n      \"name\": \"5b397553-ee61-4674-b26e-e3cf8ae2dc9d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T05:39:36.012772677Z\",\n      \"endTime\": \"2026-09-17T05:58:42.168749593Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2100766832767012864","occurred_at":"2026-09-18T02:00:17Z","signature_id":"16ebc00197f5d484a478287dbcf774e95611e6748652e8b6528ea5b7d11de1a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100452501269516288","occurred_at":"2026-09-17T05:11:15Z","signature_id":"573bf63f25f718e6e10d14bc2ae7830ca909cb9577c72529929dab5db0a4d7ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100452501269516288","occurred_at":"2026-09-17T05:11:15Z","signature_id":"cc190a2c1395e4f5ad1b156f9318c22909f9bd03dbf07b404cb1ad51d8dfd21e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100452501269516288","occurred_at":"2026-09-17T05:11:15Z","signature_id":"d97b1d417fbed487ffa515ee9e94f38afd2685c1f5cc55df9fad23cc5afc6bc9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"18ab22781017136dd055b840d502183c7edf1caa41f67dcadf9fd0a7857c8081","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":5,"after_last_push_count":5,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,5,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001d4bc80\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-56f2xp2w67w8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001d4bc80\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-56f2xp2w67w8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012fd950\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-57279mxs72dc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012fd950\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-57279mxs72dc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f2fd40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-g8sp2mm6ks67/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f2fd40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-g8sp2mm6ks67/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"3799930be8939ca48534a28b262ff33328138426e7cec1b9fb2793e6f4a95899","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"474ef928e4bfaa163ff38e770c6435689354cdc8821b9a05ec4272d8ad60c351","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"7a42cc1723b1b8bf9843f1aa12dd8baec4448f6917d7f084b3a4b154baa63afa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"8be57f60e7cd91c124c72d6d01098c6e1b8b853355d51b79f6b1bd55c240efdb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"bd4c07c42b25870c0f1baaecfa45c3c668afd2f7347cf758e572515c16e421fd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":4,"after_last_push_count":4,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,4,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004d4640\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-c8fd7f\" in resourcegroup=\"rg-np-version-upgrade-c8fd7f-kdnr5znbw57r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-c8fd7f-kdnr5znbw57r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-c8fd7f/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004d4640\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-c8fd7f\" in resourcegroup=\"rg-np-version-upgrade-c8fd7f-kdnr5znbw57r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-c8fd7f-kdnr5znbw57r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-c8fd7f/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002c1a920\u003e: \n    failed to create NodePool np-pbnkzl: failed starting nodepool creation \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\" in resourcegroup=\"rg-nightly-4-20-6c9ns9nbgrzm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-6c9ns9nbgrzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-pbnkzl/nodePools/np-pbnkzl\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002c1a920\u003e: \n    failed to create NodePool np-pbnkzl: failed starting nodepool creation \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\" in resourcegroup=\"rg-nightly-4-20-6c9ns9nbgrzm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-6c9ns9nbgrzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-pbnkzl/nodePools/np-pbnkzl\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000facdc0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-m58t8j\" resourcegroup=\"rg-np-dg-minor-m58t8j-trxh29mwfdlb\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-m58t8j-trxh29mwfdlb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-m58t8j/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000facdc0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-m58t8j\" resourcegroup=\"rg-np-dg-minor-m58t8j-trxh29mwfdlb\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-m58t8j-trxh29mwfdlb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-m58t8j/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"3d8d555b219d0ed8c9661c956fba32429ade34240dfb89932e7b6f838ec71e56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"51c0a7fe19269071bf4a86bc72b032196ba0e6e4e142e092ab1f3aca5f282e06","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"a3bcc0cf85a56ec12c072c420e0931c744e3a86ca315e7af1b9dd0b787fa5b91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"c259628d57d4675c02d0130322edaa375c878722b676fb3290dbafba51543aa4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"5c1e65a74237eb97d61c283de75bcde39935ce10b1226f1bc067c35315e03a3d","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b5a80\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-2wnkdzqz6d9w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2wnkdzqz6d9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b5a80\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-2wnkdzqz6d9w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2wnkdzqz6d9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"5c2b3fa28b82ccf14825b20d7135a7cb3d0c7f8bdbad597c1b9eba4c000e0333","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"ab5d80437500581ad7ce3942572aca8a7c5ea9eda26f4367215f73be1501d7a0","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"npupgrade-4-21\" in cluster=\"np-version-upgrade-cluster-nl8qnf\" resourcegroup=\"rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-21 in cluster np-version-upgrade-cluster-nl8qnf in resource group rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.33\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002292870\u003e: \n    failed waiting for nodepool=\"npupgrade-4-21\" in cluster=\"np-version-upgrade-cluster-nl8qnf\" resourcegroup=\"rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-21 in cluster np-version-upgrade-cluster-nl8qnf in resource group rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.33\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002292870\u003e: \n    failed waiting for nodepool=\"npupgrade-4-21\" in cluster=\"np-version-upgrade-cluster-nl8qnf\" resourcegroup=\"rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-21 in cluster np-version-upgrade-cluster-nl8qnf in resource group rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2100766832767012864","occurred_at":"2026-09-18T02:00:17Z","signature_id":"ea0f58ce05963824cd5712a4a308420855630745ad3e354951e106ba980a6661","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"9f24424c672066cba3c7cb45f333ebe45b01b356fab007f5a65bee0f062d44cf","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["INT"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:148]: failed to verify cluster \"cp-ystream-upgrade-4-22-j5vkzr\" is viable before upgrade\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015d2858\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:148]: failed to verify cluster \"cp-ystream-upgrade-4-22-j5vkzr\" is viable before upgrade\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015d2858\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100534031866138624","occurred_at":"2026-09-17T10:35:13Z","signature_id":"aac82bf00f815d526a50ddb15d7fc450980f914b55421ea2243e84387908cb3a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-11..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0016e5a88\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0016e5a88\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"77617baa5bf45330c8ac3ac9717b66793245c58ebcc17c8a3e357cf43d6f0bf5","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}