{"meta":{"start_date":"2026-09-18","end_date":"2026-09-18","start_at":"2026-09-18T00:00:00Z","end_at":"2026-09-19T00:00:00Z","timezone":"UTC","generated_at":"2026-10-14T11:02:44Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":84,"failed_runs":59,"raw_occurrences":120,"matched_occurrences":110,"runs_affected":49},"rows":[{"environment":"dev","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":19,"runs_affected":11,"failed_runs":11,"run_impact_percent":13.095238095238095,"anchor_occurrences":19,"after_last_push_count":59,"trend_counts":[0,0,0,0,71,97,19],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":76,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce8120\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-4k8qcm9nwvt9\" to finish creating: GET https://rp.j9556480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ad50471-887d-48c6-8d40-87da15772f0f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ad50471-887d-48c6-8d40-87da15772f0f\",\n \"name\": \"8ad50471-887d-48c6-8d40-87da15772f0f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:24:52.179447145Z\",\n \"endTime\": \"2026-09-18T17:53:56.374800037Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m13s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce8120\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-4k8qcm9nwvt9\" to finish creating: GET https://rp.j9556480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ad50471-887d-48c6-8d40-87da15772f0f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ad50471-887d-48c6-8d40-87da15772f0f\",\n \"name\": \"8ad50471-887d-48c6-8d40-87da15772f0f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:24:52.179447145Z\",\n \"endTime\": \"2026-09-18T17:53:56.374800037Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m13s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fae660\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-77t5pbg7l7p6\" to finish creating: GET https://rp.j2056320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9382644d-58f7-4d28-9aa1-a5cbc03fae7d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9382644d-58f7-4d28-9aa1-a5cbc03fae7d\",\n \"name\": \"9382644d-58f7-4d28-9aa1-a5cbc03fae7d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T12:53:37.696598816Z\",\n \"endTime\": \"2026-09-18T13:22:40.932765907Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m26s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fae660\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-77t5pbg7l7p6\" to finish creating: GET https://rp.j2056320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9382644d-58f7-4d28-9aa1-a5cbc03fae7d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9382644d-58f7-4d28-9aa1-a5cbc03fae7d\",\n \"name\": \"9382644d-58f7-4d28-9aa1-a5cbc03fae7d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T12:53:37.696598816Z\",\n \"endTime\": \"2026-09-18T13:22:40.932765907Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m26s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015787a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7x7svb59cvl9\" to finish creating: GET https://rp.j2056320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/79d978b9-66a2-443c-bcf8-f3536818238d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/79d978b9-66a2-443c-bcf8-f3536818238d\",\n \"name\": \"79d978b9-66a2-443c-bcf8-f3536818238d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T12:55:05.057839777Z\",\n \"endTime\": \"2026-09-18T13:24:10.88915368Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m15s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015787a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7x7svb59cvl9\" to finish creating: GET https://rp.j2056320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/79d978b9-66a2-443c-bcf8-f3536818238d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/79d978b9-66a2-443c-bcf8-f3536818238d\",\n \"name\": \"79d978b9-66a2-443c-bcf8-f3536818238d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T12:55:05.057839777Z\",\n \"endTime\": \"2026-09-18T13:24:10.88915368Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m15s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","signature_id":"99f885a8a863b5de08d9591137eee29d3c46e0095bea1b9e8046d833c161bfec","pr_number":7010,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"524cdf887bee162758b9f6eca5510d2578bb6ecf654e9948629c9452eaf1782b","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"65b5dbb9967304eedefd97b212163b3f8af9b0b2bb0188919a3d12f3a6538fc2","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","signature_id":"78800d7f9bd7ce60d82097401bbc1a1b346c5eb770d7eeedc635c6d48abac870","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","signature_id":"ba73d1a54ec8e3a0b02523dbd4276ff942ead7e6f9b4e52703889f225a290272","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100867050745368576","occurred_at":"2026-09-18T08:38:28Z","signature_id":"2e53b9a5382ed0f0493cfc0c48e9bf7aa226835370880aa8b0e3c984f3d43f8d","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100867050745368576","occurred_at":"2026-09-18T08:38:28Z","signature_id":"bb9a6198465bbe86bad7f0728097a5e8139953db856c6c8aad613b2d535d245f","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","signature_id":"001338be47cf8dcaa3fc3c500d6a6dec5a631054e1ca9f9d544c49d235b2dcb7","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","signature_id":"782ec114fda729c308336015b9ba3ea7dba7299080a2ca46beda618ff211a392","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","signature_id":"abcb5fca9b88f5de84bce4c63659906c7070748eea55c112da30810936b0fc97","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","signature_id":"82775d4e1d46d1566b3f4eab5619ed61b625107edb899f3a6d4e9d4864e726a1","pr_number":6984,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","signature_id":"925fc6ad3116b3a4af4d42c985323051e3b386c99d2bbff7d1917ed9c6b7ef67","pr_number":6984,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","signature_id":"a2d0482738306402a9aee374779b77d199fad57d181dc39a001fc3fa8cf50f3a","pr_number":6984,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","signature_id":"b838f89ca9ff0822f2ef510e9178d1d320d4033a5d5cd3ca5c98537110704efb","pr_number":6972,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780897212043264","occurred_at":"2026-09-18T02:56:11Z","signature_id":"c3a814dc2b1fc08b7b4fdb53010472ca317b4e82bd862888bcc25a0f8b4091ea","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","signature_id":"2e205df2fc76c59d7d5bced445a9af5fdbd8ffc0a4321fbfafdc448b6041ea0f","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","signature_id":"993ed1b03e808393e377627c70564f00855cc6cb2d99bd1882ef2f7b7c73270e","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","signature_id":"a44cbebfe0aee283a67f51149529d8530cd9c5797c013288bce57d67f21ed3fb","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780819697111040","occurred_at":"2026-09-18T02:55:52Z","signature_id":"a093a69ee54b8dd613584cc9b286724efd055bbbaf92d5a169de74315234601e","pr_number":7000,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":15,"runs_affected":15,"failed_runs":15,"run_impact_percent":17.857142857142858,"anchor_occurrences":15,"after_last_push_count":35,"trend_counts":[0,0,0,0,30,31,15],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":76,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":15}],"full_error_samples":["alert fired 13 time(s) (1 unknown, 12 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T20:11:59Z\n Ended: 2026-09-18T20:26:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6109056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stkk28pfi15i802bl7aop2360tp9eph-a9i6s2l5j1h0r1s\", pod=\"router-8444b6f56d-w6qnw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stkk28pfi15i802bl7aop2360tp9eph-a9i6s2l5j1h0r1s/router-8444b6f56d-w6qnw has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T17:37:21Z\n Ended: 2026-09-18T18:05:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9556480-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stibp3tbntp3l84dd3nd1p1jo0f3edi-private-ingress\", pod=\"router-8495fb9c6b-n57p2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stibp3tbntp3l84dd3nd1p1jo0f3edi-private-ingress/router-8495fb9c6b-n57p2 has been in a non-ready state for longer than 5 minutes.","alert fired 3 time(s) (2 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T13:05:51Z\n Ended: 2026-09-18T13:34:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2056320-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2steckt12k3f6paulmdgtiackkj5k84r-j7s3j8r2u9z3h1j\", pod=\"router-86c7fcbd7d-pjgqb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2steckt12k3f6paulmdgtiackkj5k84r-j7s3j8r2u9z3h1j/router-86c7fcbd7d-pjgqb has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T13:07:43Z\n Ended: 2026-09-18T13:35:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2056320-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2stedamranrq9pqrbs02e1rt645clovm-cilium-cluster\", pod=\"router-7c5f44b8f5-2rxm6\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2stedamranrq9pqrbs02e1rt645clovm-cilium-cluster/router-7c5f44b8f5-2rxm6 has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023746776109056","occurred_at":"2026-09-18T19:01:10Z","signature_id":"4beb77cb7f662a3ba0c82df6677c24959b741d146512bce1d7d72b33bd6e9a0e","pr_number":7010,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","signature_id":"adf91e76f1e8a11039895567a05fb0e5c3f1d56a9565f8cf0e71b7f1f6e5d0ce","pr_number":7010,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"991b7eb9b2760771a64ba10515aeb5d1b17b39f0d0d776abbc32c3f1fb85abb3","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100887669335658496","occurred_at":"2026-09-18T10:00:26Z","signature_id":"c3bce50243ff744e90a9975f976bf7851614b19682c44e007ffa00d7c6f101f9","pr_number":6999,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100886446859620352","occurred_at":"2026-09-18T09:55:36Z","signature_id":"fcdb5ce6d5eb5426a483be7faf7e19f591fbf253b3ebf7b8ca13d6e2fd0d11f4","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100885599773790208","occurred_at":"2026-09-18T09:52:14Z","signature_id":"002a8aff34d1219bae02653dd6f6335a032c8c53538dc0e88e47e06a4653de19","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100877868748771328","occurred_at":"2026-09-18T09:21:30Z","signature_id":"7eefe8fce4efeddab6a7bddfd5f7e5cb50289671f56d6e25708c25b9305ce796","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","signature_id":"be273c1646946db878b565bca63154b691913899e238a33aee949d4d46ae8e4f","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100867050745368576","occurred_at":"2026-09-18T08:38:28Z","signature_id":"d6db3f97bdaa9f3b8874f62b7846be7bdd694b5e229a6dfe93f7bd212a0dcb02","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","signature_id":"f47f0b6edf615e207ea20e3174c9c4b6c3860a8cd322039375d46696bfbe5705","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841559414018048","occurred_at":"2026-09-18T06:57:14Z","signature_id":"7b986d6b9e982cdf65d80f902e9563f9d1125a3f54f3c1a12116db416ae5f10d","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827581187100672","occurred_at":"2026-09-18T06:01:39Z","signature_id":"56136cc3eeacd2663934d686bb2d9b29fb8b76aaa1ea7443d05087ccb6f12b10","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100812371122458624","occurred_at":"2026-09-18T05:01:15Z","signature_id":"e85df63033962c2849bfc318fc85b0a8cba9948dc2466a5fa45b6b2724994bfb","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","signature_id":"4750aa456ebf384aa187370c588c7897567d97246241f2ecee3733d106d2cd85","pr_number":6984,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780897212043264","occurred_at":"2026-09-18T02:56:11Z","signature_id":"a0cd1baa14d1b2c8e6eaa3868608b05d0234717e1c141c4d01a43194b1743c96","pr_number":6898,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":11,"runs_affected":11,"failed_runs":11,"run_impact_percent":13.095238095238095,"anchor_occurrences":11,"after_last_push_count":13,"trend_counts":[0,0,0,0,8,7,11],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":26,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":11}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T23:47:51Z\n Ended: 2026-09-19T00:19:53Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8441088-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8441088-svc).","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T21:03:41Z\n Ended: 2026-09-18T21:09:42Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6750464-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6750464-svc).","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T17:56:41Z\n Ended: 2026-09-18T18:02:41Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9556480-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9556480-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7017/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101084611508441088","occurred_at":"2026-09-18T23:03:02Z","signature_id":"86c45d947e908dd24da56e029e4fb3fb50ede5048c717aadd07439573c0a5e5a","pr_number":7017,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101036649826750464","occurred_at":"2026-09-18T19:52:27Z","signature_id":"f8ce7b6dbb00b186dd3690b8bd0cde21b1999ed29a53641a6363be8c9bc8013e","pr_number":6945,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","signature_id":"1522dde0b9bb5d85e613b7efec8820e9ced25e3c11ff2b7a1c60a5642910649a","pr_number":7010,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6698/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100984946473570304","occurred_at":"2026-09-18T16:10:25Z","signature_id":"f7df40a156b9e56e6a5ddf150c7ac6e35049cd21802df2d33f93003f72173db8","pr_number":6698,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100987718002544640","occurred_at":"2026-09-18T16:01:55Z","signature_id":"db6a9f6935fcb92fd2604ae71dc7832d2001b5cc7b8edb8be0b3faebabe391c0","pr_number":7002,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100937243521716224","occurred_at":"2026-09-18T13:17:26Z","signature_id":"d6f983561c2168d8dc1fb0bb24a4ce58dae5f6ea67b850648a086ef0cc8ced1e","pr_number":7002,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100885599773790208","occurred_at":"2026-09-18T09:52:14Z","signature_id":"4e84895295e21c9ae798ddae187f381094acb88d5bb19172060995a42efdfbcd","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841605895294976","occurred_at":"2026-09-18T06:57:25Z","signature_id":"0d95d8184561f593338e86b1c11b5ad3f6442e5263869a130eac56ca5e35b079","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100841559414018048","occurred_at":"2026-09-18T06:57:14Z","signature_id":"b8ef7bc40a5e42cb1d9a25c2c491ed35aa8060746d13168cf09c7ed965a89066","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","signature_id":"49184b986d87df135df2fc3dcc64465876a9918af7c03bd147e1ea6126eccebf","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100779821045911552","occurred_at":"2026-09-18T02:51:54Z","signature_id":"04a95f9ed56a68aa95bb9d5ccdef76442ca4969785b15ebf86b48f429dfe945c","pr_number":6940,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":4.761904761904762,"anchor_occurrences":4,"after_last_push_count":8,"also_in":["PROD","STG"],"trend_counts":[0,0,0,0,6,6,4],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":16,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd1600\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bjbkvcsbld8c\" to finish creating: GET https://rp.j0272768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/641e2016-08be-40cf-9f8e-0e897bdccd1d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/641e2016-08be-40cf-9f8e-0e897bdccd1d\",\n \"name\": \"641e2016-08be-40cf-9f8e-0e897bdccd1d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T20:46:36.342825036Z\",\n \"endTime\": \"2026-09-18T21:05:43.553710506Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd1600\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bjbkvcsbld8c\" to finish creating: GET https://rp.j0272768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/641e2016-08be-40cf-9f8e-0e897bdccd1d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/641e2016-08be-40cf-9f8e-0e897bdccd1d\",\n \"name\": \"641e2016-08be-40cf-9f8e-0e897bdccd1d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T20:46:36.342825036Z\",\n \"endTime\": \"2026-09-18T21:05:43.553710506Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100a760\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ll6m5x\" in resourcegroup=\"rg-np-version-upgrade-ll6m5x-2g2kcltq9hpv\" to finish creating: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7e813a9-2095-43ef-9c86-073d007f6602\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7e813a9-2095-43ef-9c86-073d007f6602\",\n \"name\": \"a7e813a9-2095-43ef-9c86-073d007f6602\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:07:49.634853409Z\",\n \"endTime\": \"2026-09-18T07:26:53.294963336Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100a760\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ll6m5x\" in resourcegroup=\"rg-np-version-upgrade-ll6m5x-2g2kcltq9hpv\" to finish creating: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7e813a9-2095-43ef-9c86-073d007f6602\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a7e813a9-2095-43ef-9c86-073d007f6602\",\n \"name\": \"a7e813a9-2095-43ef-9c86-073d007f6602\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:07:49.634853409Z\",\n \"endTime\": \"2026-09-18T07:26:53.294963336Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013f8e20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-9ntdl6mdb8xb\" to finish creating: GET https://rp.j8102400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f794a701-689e-4889-b351-a1feb52109b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f794a701-689e-4889-b351-a1feb52109b7\",\n \"name\": \"f794a701-689e-4889-b351-a1feb52109b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:08.560808434Z\",\n \"endTime\": \"2026-09-18T06:32:13.567780508Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013f8e20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-9ntdl6mdb8xb\" to finish creating: GET https://rp.j8102400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f794a701-689e-4889-b351-a1feb52109b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f794a701-689e-4889-b351-a1feb52109b7\",\n \"name\": \"f794a701-689e-4889-b351-a1feb52109b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:08.560808434Z\",\n \"endTime\": \"2026-09-18T06:32:13.567780508Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101033546570272768","occurred_at":"2026-09-18T19:39:52Z","signature_id":"50e9ecb754e15646a314b5b99f973f998a45c92fb4fac3293bbdba2c5f6a2e7e","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","signature_id":"79534399cf979be34c0ffb48ddc762d6841c4a16f58dc222924b468f595a7adb","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6962/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100817340928102400","occurred_at":"2026-09-18T05:20:59Z","signature_id":"fd810f6089fd17b9ae89f5ca382f4365705ce74f6259d695bca5e674d7783e53","pr_number":6962,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6813/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815351901065216","occurred_at":"2026-09-18T05:13:05Z","signature_id":"1b937eb2ed1eae4fc31ae0598f8de6ce2076a500845fc3ea7dcd762870526b2e","pr_number":6813,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":4.761904761904762,"anchor_occurrences":4,"after_last_push_count":7,"trend_counts":[0,0,0,0,5,1,4],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":10,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":4}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T23:50:30Z\n Ended: 2026-09-19T00:17:29Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T23:50:30Z\n Ended: 2026-09-19T00:18:33Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T13:14:45Z\n Ended: 2026-09-18T13:44:43Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-gdfgt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T13:14:45Z\n Ended: 2026-09-18T13:45:42Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-gdfgt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T10:27:52Z\n Ended: 2026-09-18T10:54:52Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8771328-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.161:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8771328/providers/microsoft.monitor/accounts/hcps-j8771328\", namespace=\"ocm-arohcpci01-2stbulpvmbabhuosdje4290vg8cio72d-j5i4m0f4c4k3j8e\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-nsf7x\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stbulpvmbabhuosdje4290vg8cio72d-j5i4m0f4c4k3j8e/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T10:28:56Z\n Ended: 2026-09-18T10:54:52Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8771328-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.161:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8771328/providers/microsoft.monitor/accounts/hcps-j8771328\", namespace=\"ocm-arohcpci01-2stbulpvmbabhuosdje4290vg8cio72d-j5i4m0f4c4k3j8e\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-nsf7x\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2stbulpvmbabhuosdje4290vg8cio72d-j5i4m0f4c4k3j8e/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","signature_id":"7dead5f17d20c600ef9a15a67c9915db561b768a43db7c68ada9d07b7b90dfc6","pr_number":7012,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"77db7532989dc335d44e3fd0e347138b7a4aa37045fd816ecfc3761a0789b616","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100877868748771328","occurred_at":"2026-09-18T09:21:30Z","signature_id":"077ee51cbdfb7388e2c8c28578e14f7c26aad49cf417b4ab9a95eda6e094ce1d","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","signature_id":"32caccfe24784ea73efa24f662aed9cbdd2ff753b7f3457b2fae68c38d3d3c3d","pr_number":6917,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":4.761904761904762,"anchor_occurrences":4,"after_last_push_count":3,"trend_counts":[0,0,0,0,3,1,4],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":4}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T09:27:59Z\n Ended: 2026-09-18T09:34:55Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6818048-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T09:23:28Z\n Ended: 2026-09-18T09:39:34Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8030592-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T04:18:48Z\n Ended: 2026-09-18T04:37:50Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2362240-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100862764498030592","occurred_at":"2026-09-18T08:21:29Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784091786907648","occurred_at":"2026-09-18T03:08:52Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":6984,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.5714285714285716,"anchor_occurrences":3,"after_last_push_count":8,"also_in":["PROD"],"trend_counts":[0,0,0,0,7,1,3],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":11,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b4b300\u003e: \n failed to create HCP cluster ea-listtxmfvt: failed waiting for cluster=\"ea-listtxmfvt\" in resourcegroup=\"ea-list-b22vvl6sfzgh\" to finish creating: GET https://rp.j8565632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80b95276-553d-4fcb-9838-22d1a7553049\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80b95276-553d-4fcb-9838-22d1a7553049\",\n \"name\": \"80b95276-553d-4fcb-9838-22d1a7553049\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T23:30:18.658850049Z\",\n \"endTime\": \"2026-09-18T23:59:19.341659624Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b4b300\u003e: \n failed to create HCP cluster ea-listtxmfvt: failed waiting for cluster=\"ea-listtxmfvt\" in resourcegroup=\"ea-list-b22vvl6sfzgh\" to finish creating: GET https://rp.j8565632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80b95276-553d-4fcb-9838-22d1a7553049\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80b95276-553d-4fcb-9838-22d1a7553049\",\n \"name\": \"80b95276-553d-4fcb-9838-22d1a7553049\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T23:30:18.658850049Z\",\n \"endTime\": \"2026-09-18T23:59:19.341659624Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001084500\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-95vjt5mdwdlk\" to finish creating: GET https://rp.j8771328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ec69745-9888-4c12-ad3c-cfe984e0f2d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ec69745-9888-4c12-ad3c-cfe984e0f2d3\",\n \"name\": \"0ec69745-9888-4c12-ad3c-cfe984e0f2d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T10:07:19.283105586Z\",\n \"endTime\": \"2026-09-18T10:36:28.169644011Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001084500\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-95vjt5mdwdlk\" to finish creating: GET https://rp.j8771328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ec69745-9888-4c12-ad3c-cfe984e0f2d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ec69745-9888-4c12-ad3c-cfe984e0f2d3\",\n \"name\": \"0ec69745-9888-4c12-ad3c-cfe984e0f2d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T10:07:19.283105586Z\",\n \"endTime\": \"2026-09-18T10:36:28.169644011Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--xq7n5gbztqbj/cluster-hs-pre-kf6554 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001090320\u003e: \n failed to create HCP cluster cluster-hs-pre-kf6554: failed waiting for cluster=\"cluster-hs-pre-kf6554\" in resourcegroup=\"rg-hs-pre--xq7n5gbztqbj\" to finish creating: GET https://rp.j2362240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359ebfce-873d-4d6c-8f64-6e12fb114150\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359ebfce-873d-4d6c-8f64-6e12fb114150\",\n \"name\": \"359ebfce-873d-4d6c-8f64-6e12fb114150\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T04:02:43.525962262Z\",\n \"endTime\": \"2026-09-18T04:31:43.022568987Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--xq7n5gbztqbj/cluster-hs-pre-kf6554 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001090320\u003e: \n failed to create HCP cluster cluster-hs-pre-kf6554: failed waiting for cluster=\"cluster-hs-pre-kf6554\" in resourcegroup=\"rg-hs-pre--xq7n5gbztqbj\" to finish creating: GET https://rp.j2362240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359ebfce-873d-4d6c-8f64-6e12fb114150\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/359ebfce-873d-4d6c-8f64-6e12fb114150\",\n \"name\": \"359ebfce-873d-4d6c-8f64-6e12fb114150\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T04:02:43.525962262Z\",\n \"endTime\": \"2026-09-18T04:31:43.022568987Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","signature_id":"2c332da578489a812e0144aacf458767c886ed8769b126097a13ac0573609822","pr_number":7012,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100877868748771328","occurred_at":"2026-09-18T09:21:30Z","signature_id":"f590a6d9decf3d5d50d5dfcef242c5c8af6ebdbb758d3e90d6fae5136781bae3","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","signature_id":"89409b5febc6093a9cefc9b6ca32afd019b45b211957e3bd23fe77c3eda373f3","pr_number":6917,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.5714285714285716,"anchor_occurrences":3,"after_last_push_count":6,"trend_counts":[0,0,0,0,6,6,3],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":15,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-19T00:04:30Z\n Ended: 2026-09-19T00:23:35Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 6 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-18T18:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti77pkq7agiit4e9p66ic3479hnrq5-a7o9q7y9a3g4n2w\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti77pkq7agiit4e9p66ic3479hnrq5-a7o9q7y9a3g4n2w/router has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-18T18:03:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti77pkq7agiit4e9p66ic3479hnrq5-a7o9q7y9a3g4n2w\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti77pkq7agiit4e9p66ic3479hnrq5-a7o9q7y9a3g4n2w/router has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-18T18:11:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-18T18:11:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-09-18T18:12:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-09-18T18:12:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1089664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/hcps-j1089664\", namespace=\"ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sti7a39k8no3d3k6o47iqafudkp0ufb-sre-hcp-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T13:28:41Z\n Ended: 2026-09-18T13:42:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-gdfgt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-18T13:28:41Z\n Ended: 2026-09-18T13:42:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-gdfgt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/capi-provider has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","signature_id":"61a9d86a254d9ba501b68a3f3cb4e321b30d8b314bb25ec5b72e60af5be20fd0","pr_number":7012,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"035123031fa1d20ace889ddb3bd473206d7c64b2e0e69a0775e5e52e6c40e898","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"b30257b5749c88e7acf4c108618d88d0f18563968e68f249086f8fef88556ee6","pr_number":6933,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.5714285714285716,"anchor_occurrences":3,"after_last_push_count":5,"trend_counts":[0,0,0,0,2,1,3],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":3}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T23:52:29Z\n Ended: 2026-09-19T00:21:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e1a1529f-88f5-4f8c-8503-a2513cf48144\"\n Description: Pod ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-09-18T23:53:28Z\n Ended: 2026-09-19T00:17:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/hcps-j8565632\", namespace=\"ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e1a1529f-88f5-4f8c-8503-a2513cf48144\"\n Description: Pod ocm-arohcpci01-2stnn2qj5n4300vnra0q4o64lcoi8994-ea-listtxmfvt/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T13:16:41Z\n Ended: 2026-09-18T13:44:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"af416a40-e08e-480e-bf5b-da79914398be\"\n Description: Pod ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-09-18T13:17:41Z\n Ended: 2026-09-18T13:45:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2056320-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2056320/providers/microsoft.monitor/accounts/hcps-j2056320\", namespace=\"ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"af416a40-e08e-480e-bf5b-da79914398be\"\n Description: Pod ocm-arohcpci01-2stecjn5ajol8jdm27uamrva75b6nf2v-r5f9v1s9k2k3j8y/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T04:31:05Z\n Ended: 2026-09-18T04:48:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2362240-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2362240/providers/microsoft.monitor/accounts/hcps-j2362240\", namespace=\"ocm-arohcpci01-2st6jp6ln5eij992a4h7v8fe6stngjn4-s9g6g5z2i0s8b2f\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e333cab1-7b1b-43d4-a2a8-c38ce512c9fe\"\n Description: Pod ocm-arohcpci01-2st6jp6ln5eij992a4h7v8fe6stngjn4-s9g6g5z2i0s8b2f/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","signature_id":"b0cb1249b73e4e17694cee1167aadd3f0fe5951d24186dea9efdb1984ff09dea","pr_number":7012,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"f71f9d0966fc26ad6ff3cd9a2f086d86818c4234b98d4c027fd22b942a6e7ba2","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100787332612362240","occurred_at":"2026-09-18T03:21:45Z","signature_id":"b14a5290094a9e0d25292157800f9158e06c44b980a0e58131fba650ea86837b","pr_number":6917,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.5714285714285716,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T22:08:21Z\n Ended: 2026-09-18T22:18:16Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j4790528-svc\", component=\"frontend\", method=\"delete\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j4790528-svc).","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T18:04:41Z\n Ended: 2026-09-18T18:24:40Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j9556480-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j9556480-svc).","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T13:28:52Z\n Ended: 2026-09-18T13:38:52Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j6576768-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j6576768-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7018/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101049380864790528","occurred_at":"2026-09-18T20:43:02Z","signature_id":"900336073fa1807e3fa5eb270059d6e459df8256995e512e7d9ae978fff82f8c","pr_number":7018,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100988487489556480","occurred_at":"2026-09-18T16:36:50Z","signature_id":"f6e1d34656f1ce37365020fae4967ac3b4ae64788fb876cb79e6b050b731b2fe","pr_number":7010,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100917027756576768","occurred_at":"2026-09-18T11:57:07Z","signature_id":"85fef6c172f8238e32065ce9be94996092fd65499dbb48afe109749b4511663e","pr_number":6939,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.5714285714285716,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[0,0,0,0,2,0,3],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.009s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.009s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101053926852792320","occurred_at":"2026-09-18T21:00:47Z","signature_id":"453c681e05619434e546781762e4d8f81384d3fa3b09f4cc9d65ac2b7dc22b1d","pr_number":6639,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827581187100672","occurred_at":"2026-09-18T06:01:39Z","signature_id":"ee111baab05f6b8ee26f353e865437e1bffda7140d9fc154398f141cd776e6f7","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6813/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815351901065216","occurred_at":"2026-09-18T05:13:05Z","signature_id":"453c681e05619434e546781762e4d8f81384d3fa3b09f4cc9d65ac2b7dc22b1d","pr_number":6813,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.5714285714285716,"anchor_occurrences":3,"after_last_push_count":4,"trend_counts":[0,0,0,0,2,1,3],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001a0ed40\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59914-\u003e172.30.0.10:53: read: connection refused\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001a0ed40\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59914-\u003e172.30.0.10:53: read: connection refused\n ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.002s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000c8c390\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:35668-\u003e172.30.0.10:53: read: connection refused\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.002s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000c8c390\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:35668-\u003e172.30.0.10:53: read: connection refused\n ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0018c70e0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59154-\u003e172.30.0.10:53: read: connection refused\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0018c70e0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:59154-\u003e172.30.0.10:53: read: connection refused\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7003/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100859139801485312","occurred_at":"2026-09-18T08:07:05Z","signature_id":"4553dd78f39dc989ce886fb5e82604c7aa8be953927622430a5564e21a9cc7f0","pr_number":7003,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100827581187100672","occurred_at":"2026-09-18T06:01:39Z","signature_id":"25746549fcd7e3d493a2e9010aa4c2f674167bdbce01dd388ff30ebecae482d7","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780868975988736","occurred_at":"2026-09-18T02:56:04Z","signature_id":"3f4f2a1f835d80221855e8810686dfbc430d2c91ebb857edce8074813fea620c","pr_number":6939,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.380952380952381,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0017e8150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-5mwmp59kkrll\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\",\n \"name\": \"0000780c-679b-479e-b61c-e08211991339\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:27:09.508692971Z\",\n \"endTime\": \"2026-09-18T17:51:11.117212687Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti77pkq7agiit4e9p66ic3479hnrq5 still exists (deletion dispatched at 2026-09-18T17:27:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0017e8150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-5mwmp59kkrll\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0000780c-679b-479e-b61c-e08211991339\",\n \"name\": \"0000780c-679b-479e-b61c-e08211991339\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:27:09.508692971Z\",\n \"endTime\": \"2026-09-18T17:51:11.117212687Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti77pkq7agiit4e9p66ic3479hnrq5 still exists (deletion dispatched at 2026-09-18T17:27:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019d60f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vmz65bf72428\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n \"name\": \"e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:07:58.803605122Z\",\n \"endTime\": \"2026-09-18T07:32:03.445509106Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95msfbb4nhua5a6pq8m3044dhe0se still exists (deletion dispatched at 2026-09-18T07:08:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019d60f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vmz65bf72428\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n \"name\": \"e838e2dd-c8ac-447c-b2c0-095cbbe04623\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:07:58.803605122Z\",\n \"endTime\": \"2026-09-18T07:32:03.445509106Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95msfbb4nhua5a6pq8m3044dhe0se still exists (deletion dispatched at 2026-09-18T07:08:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c80150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nglkjctcmb9p\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n \"name\": \"9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:10:06.620770558Z\",\n \"endTime\": \"2026-09-18T07:34:13.53272476Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95o86ftkou4io40i8o12g13uerlpd still exists (deletion dispatched at 2026-09-18T07:10:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c80150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nglkjctcmb9p\" to finish deleting: GET https://rp.j5412736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n \"name\": \"9724272a-25ec-4c0a-b43f-3f85230b0b68\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:10:06.620770558Z\",\n \"endTime\": \"2026-09-18T07:34:13.53272476Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st95o86ftkou4io40i8o12g13uerlpd still exists (deletion dispatched at 2026-09-18T07:10:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"52f64f30d917c25349ded7085a1164f0e2615c20f54359a71e907e0ab61c033a","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","signature_id":"8db752092d73d2ebfaaaef7b1150f759438bf639db4c2c7645ebdce228946dcf","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100829561175412736","occurred_at":"2026-09-18T06:09:33Z","signature_id":"cc794ab286f7ef163ad70de687149ee27cd59dcc48d24d280de823796404177d","pr_number":6969,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,3,2,3],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00156c060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-nzl72g5md8zn\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n \"name\": \"a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:31:17.188166352Z\",\n \"endTime\": \"2026-09-18T17:55:21.133690065Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7a39k8no3d3k6o47iqafudkp0ufb still exists (deletion dispatched at 2026-09-18T17:31:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00156c060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-nzl72g5md8zn\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n \"name\": \"a6fd0546-7edc-4a7f-8a5d-bb281048b13a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:31:17.188166352Z\",\n \"endTime\": \"2026-09-18T17:55:21.133690065Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7a39k8no3d3k6o47iqafudkp0ufb still exists (deletion dispatched at 2026-09-18T17:31:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012e61b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-ghmpzk\" in resourcegroup=\"rg-zstream-upgrade-4-20-nzh4xtf2qkhs\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\",\n \"name\": \"58c027c6-4639-48fa-ace0-f454c7358b62\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:43:42.257649954Z\",\n \"endTime\": \"2026-09-18T18:07:51.061074345Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7ae2rm414v72bb1h0rpr5prt24in still exists (deletion dispatched at 2026-09-18T17:43:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012e61b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-ghmpzk\" in resourcegroup=\"rg-zstream-upgrade-4-20-nzh4xtf2qkhs\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/58c027c6-4639-48fa-ace0-f454c7358b62\",\n \"name\": \"58c027c6-4639-48fa-ace0-f454c7358b62\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:43:42.257649954Z\",\n \"endTime\": \"2026-09-18T18:07:51.061074345Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7ae2rm414v72bb1h0rpr5prt24in still exists (deletion dispatched at 2026-09-18T17:43:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014723f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listg6frkr\" in resourcegroup=\"ea-list-r7jhwkxx4sdv\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n \"name\": \"9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:26:18.493088022Z\",\n \"endTime\": \"2026-09-18T17:50:21.112379819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7eotueh3r9mn8c08fdvae8djl7o8 still exists (deletion dispatched at 2026-09-18T17:26:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014723f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listg6frkr\" in resourcegroup=\"ea-list-r7jhwkxx4sdv\" to finish deleting: GET https://rp.j1089664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n \"name\": \"9bf9f7c9-1f78-4d84-bb7d-1c79558f8921\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T17:26:18.493088022Z\",\n \"endTime\": \"2026-09-18T17:50:21.112379819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sti7eotueh3r9mn8c08fdvae8djl7o8 still exists (deletion dispatched at 2026-09-18T17:26:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"7e8e740680a98bd62be8f0f23d65fc40251d4d5f7742370deded696800b1b588","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"9e65f4af22bf7ec9953560fed8bb49590fa82fc63d7f9f6d24d707a1a503f81c","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"a0f110873c9fc7db9f376a400bf0d8b83aa3eddcd644b62e99f4963c1f15aa39","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a6bab59ff05e72a771d612966dd1e7024c1a3d21427661ca54c8003eb2c7e563","failure_pattern":"ERROR CODE: DeploymentFailed; detail code GetMSICredentialError; detail message Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.380952380952381,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,1,0,2],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-18T03:37:37.864Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-18T03:37:38.393Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-18T03:37:39.053Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=9d03ef43fb389999fc408f90bd02e6c94ce2721f8e17a8400c8a777195fbc298 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1401344-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9d03ef43fb389999fc408f90bd02e6c94ce2721f8e17a8400c8a777195fbc298\ntime=2026-09-18T03:40:39.648Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1401344-mgmt-1/providers/Microsoft.Resources/deployments/9d03ef43fb389999fc408f90bd02e6c94ce2721f8e17a8400c8a777195fbc298/operationStatuses/08584119042267536349\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1401344-mgmt-1/providers/Microsoft.Resources/deployments/cluster-sr7huchijtjyg\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1401344-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j1401344-mgmt-1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"GetMSICredentialError\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-18T01:22:54.255Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-18T01:22:54.877Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-18T01:22:55.881Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=20d345ced78ed93f1fcc83ca672f15621e19f0c6572d4980fecd14d77cfa1a28 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9648512-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F20d345ced78ed93f1fcc83ca672f15621e19f0c6572d4980fecd14d77cfa1a28\ntime=2026-09-18T01:24:56.385Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9648512-mgmt-2/providers/Microsoft.Resources/deployments/20d345ced78ed93f1fcc83ca672f15621e19f0c6572d4980fecd14d77cfa1a28/operationStatuses/08584119123100595444\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9648512-mgmt-2/providers/Microsoft.Resources/deployments/cluster-zhmgezzmxfkbc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9648512-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j9648512-mgmt-2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"GetMSICredentialError\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6984/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100784773881401344","occurred_at":"2026-09-18T03:11:35Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6984,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100751960339648512","occurred_at":"2026-09-18T01:00:47Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6969,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410","failure_pattern":"ERROR CODE: DeploymentStackDeploymentFailed","search_query":"ERROR CODE: DeploymentStackDeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.380952380952381,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,2,0,2],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","occurrences":2}],"full_error_samples":["time=2026-09-18T21:50:35.372Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n Kind: ARMStack\\n Template: templates/monitoring.bicep\\n Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-09-18T21:51:25.579Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/04392d3b-593d-406d-ae87-240d0d6c1c70\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/04392d3b-593d-406d-ae87-240d0d6c1c70\\\",\\n \\\"name\\\": \\\"04392d3b-593d-406d-ae87-240d0d6c1c70\\\",\\n \\\"status\\\": \\\"failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'aad2e336-6c43-473c-997c-4a60a482583a'.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4937088/providers/Microsoft.Resources/deployments/y5flko475afn22dntfyewzfj7gfkbiwdmj7ezjquufwse-26091821etad3\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"Property name: Rules[0].Expression, Attempted value: null, Error: 1:237: parse error: unknown escape sequence U+002E '.' Activity ID: b8850252-9aac-448f-9581-e29b2b8a10a8.\\\"\\n }\\n ]\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-18T15:40:00.669Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n Kind: ARMStack\\n Template: templates/monitoring.bicep\\n Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-09-18T15:42:51.615Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/eastus2/deploymentStackOperationStatus/326eddf6-9965-4299-86de-7e1eb754418c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/eastus2/deploymentStackOperationStatus/326eddf6-9965-4299-86de-7e1eb754418c\\\",\\n \\\"name\\\": \\\"326eddf6-9965-4299-86de-7e1eb754418c\\\",\\n \\\"status\\\": \\\"failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'e78fa208-8c43-45b6-a80c-57ad93c6a044'.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4742016/providers/Microsoft.Resources/deployments/ka7tdccaiavlrtibmy3y5s7tfhcs6gpzxc6p4xe5pchjy-26091815bptz8\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"Property name: Rules[0].Expression, Attempted value: null, Error: 1:237: parse error: unknown escape sequence U+002E '.' Activity ID: d9b78f81-a4e0-4b15-9e4d-75e6f5c342e3.\\\"\\n }\\n ]\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101062836254937088","occurred_at":"2026-09-18T21:36:30Z","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","pr_number":6965,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100966207174742016","occurred_at":"2026-09-18T15:08:20Z","signature_id":"e4c3fb1587b09488a2265155d7a6787ca94e35b729bce621d58ebb8765ea7ef1","pr_number":6965,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.380952380952381,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,3,0,2],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001bed8b0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001bed8b0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.008s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000909420\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.008s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000909420\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101053926852792320","occurred_at":"2026-09-18T21:00:47Z","signature_id":"6566438295d27151c619c08c959579eb7709fb58c49e3c34c87dd1ec1d1e808b","pr_number":6639,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","signature_id":"7193ae4d62017e53f8c9422389a5405738a2e8a036e8555e76291f330d89920e","pr_number":6900,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.380952380952381,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[0,0,0,0,4,3,2],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":9,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100868434823745536","occurred_at":"2026-09-18T08:44:01Z","signature_id":"e3628da3891e1ec49619dde7547596c92e9e1c1cfe73fca8833c1206090506a8","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6922,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[0,0,0,0,5,3,2],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":9,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:111]: failed to create node pool \"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001015700\u003e: \n failed to create NodePool np-zrxbbn: failed waiting for nodepool=\"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\" in resourcegroup=\"rg-hs-pre--gs2tq8nbktl2\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n \"name\": \"4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:55:06.959884593Z\",\n \"endTime\": \"2026-09-18T03:55:51.664849739Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:111]: failed to create node pool \"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001015700\u003e: \n failed to create NodePool np-zrxbbn: failed waiting for nodepool=\"np-zrxbbn\" for cluster \"cluster-hs-pre-zrxbbn\" in resourcegroup=\"rg-hs-pre--gs2tq8nbktl2\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n \"name\": \"4afee37d-b341-457c-963a-1bd1b8f2a81c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:55:06.959884593Z\",\n \"endTime\": \"2026-09-18T03:55:51.664849739Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0017ae060\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-r4g2f9rh44ss\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n \"name\": \"499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:54:50.355767916Z\",\n \"endTime\": \"2026-09-18T03:55:21.65277157Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0017ae060\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-r4g2f9rh44ss\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n \"name\": \"499b13a6-9803-4f67-82be-dcf4f0fa859c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:54:50.355767916Z\",\n \"endTime\": \"2026-09-18T03:55:21.65277157Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","signature_id":"6acf378b1d751cabdd2cb29f20e478c64231e5b033214257ad51d0d59a0c2ba7","pr_number":6972,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","signature_id":"f081e78216b7bd4a54e652f3dfd3a2cf8cf19638bb76bda01151759533a92078","pr_number":6972,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"3ec8dfede2b7d12b6200fcfa62f782082fd9d3f96c215ae2c1de00861966d250","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":2,"after_last_push_count":0,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7ccc0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xp4rlvdpbgj6\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\",\n \"name\": \"b344e921-2b47-4118-aac7-81c45cb67cc0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:19.863238771Z\",\n \"endTime\": \"2026-09-18T06:42:22.545828006Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7ccc0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xp4rlvdpbgj6\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b344e921-2b47-4118-aac7-81c45cb67cc0\",\n \"name\": \"b344e921-2b47-4118-aac7-81c45cb67cc0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:19.863238771Z\",\n \"endTime\": \"2026-09-18T06:42:22.545828006Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a46a00\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jt9b8z9tj78s\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n \"name\": \"16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:10.341418654Z\",\n \"endTime\": \"2026-09-18T06:42:12.545110519Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a46a00\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jt9b8z9tj78s\" to finish creating: GET https://rp.j0748928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n \"name\": \"16ca80cc-2a38-4172-a9cc-5c6829de0ae2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T06:13:10.341418654Z\",\n \"endTime\": \"2026-09-18T06:42:12.545110519Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","signature_id":"0d8835203476a034717107aadec991b28272c94910dad48bde3bb462bb56ac9b","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","signature_id":"4fb56cfa0c5d81c59f21dba90bd2b4aeaa2b18f5202205163a7a19b83e403f4e","pr_number":6157,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"240d7e65fa5e2b6ff5542fc888978c21e203da5524e2e297822471f59f51aa10","failure_pattern":"\"error\": {","search_query":"\"error\": {","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:139]: failed to get backup schedules for cadence check\nUnexpected error:\n \u003c*errors.errorString | 0xc0010c3e40\u003e: \n expected status 200, got 412: {\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"kube-applier client not available for management cluster /providers/microsoft.redhatopenshift/stamps/1/managementclusters/default\"\n }\n }\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:139]: failed to get backup schedules for cadence check\nUnexpected error:\n \u003c*errors.errorString | 0xc0010c3e40\u003e: \n expected status 200, got 412: {\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"kube-applier client not available for management cluster /providers/microsoft.redhatopenshift/stamps/1/managementclusters/default\"\n }\n }\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","signature_id":"06ba3d2a9582d2a18869bdd5ae1df4321a4d0977a7be4bdd0e3cc1af2ec0608a","pr_number":6157,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[0,0,0,0,2,1,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6933,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-q78vhx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f621e0\u003e: \n DNS for route host agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-q78vhx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f621e0\u003e: \n DNS for route host agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s69wn.apps.aro.o5k6a1c8v4j7o5i.oz3q.j2056320.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100916460112056320","occurred_at":"2026-09-18T11:54:51Z","signature_id":"803dd3435e0189d03ce01d4d7de3f8e741a5ee356c8001e19b84595bbaa71298","pr_number":6933,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"8aa573d68ede374d56fcd25dace5feffa8644835d49bf0c7ba52006e6de9139d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentNotFound; detail message Deployment 'svc-kv-xehipxbjxawki' could not be found.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra-output","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra-output","occurrences":1}],"full_error_samples":["time=2026-09-18T18:49:43.745Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output description=\"Step infra-output\\n Kind: ARM\\n Template: templates/svc-infra-lookup.bicep\\n Parameters: configurations/svc-infra-lookup.tmpl.bicepparam\"\ntime=2026-09-18T18:49:44.412Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output\ntime=2026-09-18T18:49:45.666Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output deployment=299b33c33a8d1927824771d1411d9cc2911c3c4239fa63e4af7c6c883950aabd portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6366720-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F299b33c33a8d1927824771d1411d9cc2911c3c4239fa63e4af7c6c883950aabd\ntime=2026-09-18T18:50:16.326Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6366720-svc/providers/Microsoft.Resources/deployments/299b33c33a8d1927824771d1411d9cc2911c3c4239fa63e4af7c6c883950aabd/operationStatuses/08584118495004368821\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"NotFound\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentNotFound\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Deployment 'svc-kv-xehipxbjxawki' could not be found.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7017/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101014245486366720","occurred_at":"2026-09-18T18:23:12Z","signature_id":"09b1458b46da16205cd209ec402488a728d06d1e258b36845618ba3741f96692","pr_number":7017,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c2687040eef152546e303fe3ce6da2f92a92db4c9a3989bfdcd2df869f68897f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90b40\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8r8jd5phj8ps\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\",\n \"name\": \"de938633-4e59-468a-9378-96c48d8ef3fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:48:43.242131083Z\",\n \"endTime\": \"2026-09-18T04:17:44.681797406Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 16m33s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90b40\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8r8jd5phj8ps\" to finish creating: GET https://rp.j8628480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de938633-4e59-468a-9378-96c48d8ef3fd\",\n \"name\": \"de938633-4e59-468a-9378-96c48d8ef3fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:48:43.242131083Z\",\n \"endTime\": \"2026-09-18T04:17:44.681797406Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 16m33s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780853188628480","occurred_at":"2026-09-18T02:56:00Z","signature_id":"a6058215e786bb088f449c6f95b45c11ce369a5297dbcc0542779a89a2c66add","pr_number":6922,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,22,19,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":13,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001520108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-ptc9sq59cf2d\" to finish deleting: GET https://rp.j6053632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3852ce7c-96ee-4d91-9970-a39897a9c07a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3852ce7c-96ee-4d91-9970-a39897a9c07a\",\n \"name\": \"3852ce7c-96ee-4d91-9970-a39897a9c07a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T14:15:14.231768617Z\",\n \"endTime\": \"2026-09-18T14:39:19.156773648Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2stfc33bsiqt6ka6nm1u076ksjbrgtab still exists (deletion dispatched at 2026-09-18T14:15:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001520108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-ptc9sq59cf2d\" to finish deleting: GET https://rp.j6053632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3852ce7c-96ee-4d91-9970-a39897a9c07a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3852ce7c-96ee-4d91-9970-a39897a9c07a\",\n \"name\": \"3852ce7c-96ee-4d91-9970-a39897a9c07a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T14:15:14.231768617Z\",\n \"endTime\": \"2026-09-18T14:39:19.156773648Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2stfc33bsiqt6ka6nm1u076ksjbrgtab still exists (deletion dispatched at 2026-09-18T14:15:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100931988906053632","occurred_at":"2026-09-18T12:56:34Z","signature_id":"946ebd2cdc29e94011829e7ef21165a52d1801fd362344870c8d6a3e7c5269c4","pr_number":6999,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-8ld2f474grdt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-8ld2f474grdt, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["STG"],"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00255de60\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-8ld2f474grdt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-8ld2f474grdt, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00255de60\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-8ld2f474grdt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-8ld2f474grdt, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101014543869153280","occurred_at":"2026-09-18T18:24:36Z","signature_id":"96d0afdee41d99182089b8922944bd77cbf90a6f0b4ca2fe3e238fbfdaae440e","pr_number":6639,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"5cfe4f61cfcfa6903321f1c971ea5ab31ebee2ed754015e96c99b68dfe286ebe","failure_pattern":"VM run command \"e2e-runcommand-1789704441845701237\" on VM \"\u003cvm\u003e\" failed and cleanup also failed: Get \"\u003curl\u003e\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"\u003curl\u003e\": stream error: stream ID 33; CANCEL; received from peer)","search_query":"VM run command \"e2e-runcommand-1789704441845701237\" on VM \"cidr-connectivity-test-test-vm\" failed and cleanup also failed: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/locations/CanadaCentral/operations/67651689-75ab-4f6c-aa6e-1746fbbdc2ad?p=c6bb2de4-260b-49a6-86a7-56c69bdc0403\u0026api-version=2024-03-01\u0026t=639253012427558463\u0026c=MIIHlDCCBnygAwIBAgIQTJW7e7XbMoQlWPa331WcnDANBgkqhkiG9w0BAQsFADA2MTQwMgYDVQQDEytDQ01FIEcxIFRMUyBSU0EgMjA0OCBTSEEyNTYgMjA0OSBXVVMyIENBIDAxMB4XDTI2MDgxMTE3MzMyMloXDTI3MDIwNjIzMzMyMlowQDE-MDwGA1UEAxM1YXN5bmNvcGVyYXRpb25zaWduaW5nY2VydGlmaWNhdGUubWFuYWdlbWVudC5henVyZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeQHh5tmZ6ZYRlf_Q_PwPJqUN9t1MEgnet-7jMlsIvkZBhjMDVPNEFS1UtIM-3StxxWCffcghG31P1UcexhHelJCujRWSZYWfEIC7z_S8BBwxDhcc2peEIdNKD1O6UKvglkD8KRsPxRT-ZwQYY3qFNngixnx3EDhtWr1a9Kg46bdJJAIIzu3flYJzQzP3yVrFuRQn4iH75UvhgEByfjJaD75Ugw9JYkgO52nE5dVc2ddxoRMWzJKxp_V2Adp7VqTbMx2T8hO9PWqkltIPwoAXIubXFj3Z5A6Wj7hE7XmY_2cFJKJfspjiqwLyfnLqlv9pCLhIVdG31g8DAGZM7fc2lAgMBAAGjggSSMIIEjjCBnQYDVR0gBIGVMIGSMAwGCisGAQQBgjd7AQEwZgYKKwYBBAGCN3sCAjBYMFYGCCsGAQUFBwICMEoeSAAzADMAZQAwADEAOQAyADEALQA0AGQANgA0AC0ANABmADgAYwAtAGEAMAA1ADUALQA1AGIAZABhAGYAZgBkADUAZQAzADMAZDAMBgorBgEEAYI3ewMCMAwGCisGAQQBgjd7BAIwDAYDVR0TAQH_BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDgYDVR0PAQH_BAQDAgWgMB0GA1UdDgQWBBQbwB3CifQh36XI9i1lBe8503G1RTAfBgNVHSMEGDAWgBSs43L6A7Jznj2VyO-HW67dG6HtaDCCAbIGA1UdHwSCAakwggGlMGmgZ6BlhmNodHRwOi8vcHJpbWFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwa6BpoGeGZWh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3R1czIvY3Jscy9jY21ld2VzdHVzMnBraS9jY21ld2VzdHVzMmljYTAxLzY3L2N1cnJlbnQuY3JsMFqgWKBWhlRodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwb6BtoGuGaWh0dHA6Ly9jY21ld2VzdHVzMnBraS53ZXN0dXMyLnBraS5jb3JlLndpbmRvd3MubmV0L2NlcnRpZmljYXRlQXV0aG9yaXRpZXMvY2NtZXdlc3R1czJpY2EwMS82Ny9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jYWNlcnRzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWV3ZXN0dXMycGtpLndlc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQACHqTwaTHdHWv9uHzVkq0ltJxzjLPIQHJnP6ndMrFnY-TIxQDrABl7eSPlINC2qk1CFExKLmjYM8aDst83BXP-yH0AHIvBrOGVdZvqFE-8KPpkqE97J5NNLNr1lLkz4vWpbjmJhLvYmureoMJjtcLpQyz-EfACVM90Lkmu8ueIkp5mjzxa_-cMa3KYwbFXLO8huXYcrhLhvs31Z9czJKjlPptL0ZE1-rBlMXGqTdZUE1CfPRoTjMMRZe_jEJYNKyxhL_5fT2mLxdlYJsMDeTU5qM-x3lA74zxVrUvC0ZvCQdK629aZQUbWV74Uzk1DmAtUiDhGvnsOW-rug_uLr621\u0026s=2ED070N0ZTuw-f1HVDJ70D4TL-E7a_pmemS26JAcsDWdtZNt5piI-6t7rp4DAeUvveysVwPzvAf5ULh-g26b96AAV9pXZ4I1vo74l2qjDBTA1ho2Ia11r_3U1XAtiPoLjGufya8CYXOek1iwZu7oaFEZ-4uayFIei-jUkQWxZ9Jz8_HKfyzJZZA1qJtJWKMIu_-fMQRELNXCWOkBgzgrUj1MP39GcY4oa5lZoVmPJGhHGseHZ--ETS4Vun62lwfxl7XrPb2PN0L9g7S07Nil12FIWSbh_qVJ2DC5m-7fPlkYze7LvocmWvHPwMOLiIbvvBS9Ui4BX7syQlC25zGKXw\u0026h=siGPFN1sMGFTEzhJalRHQiDtbRbqB8eZGXQRBJUWkyo\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qhh4b47m8ncv/providers/Microsoft.Compute/virtualMachines/cidr-connectivity-test-test-vm/runCommands/e2e-runcommand-1789704441845701237?api-version=2024-03-01\": stream error: stream ID 33; CANCEL; received from peer)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:293]: failed to create cluster role binding for external auth via VM\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d173e0\u003e: \n VM run command \"e2e-runcommand-1789704441845701237\" on VM \"cidr-connectivity-test-test-vm\" failed and cleanup also failed: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/locations/CanadaCentral/operations/67651689-75ab-4f6c-aa6e-1746fbbdc2ad?p=c6bb2de4-260b-49a6-86a7-56c69bdc0403\u0026api-version=2024-03-01\u0026t=639253012427558463\u0026c=MIIHlDCCBnygAwIBAgIQTJW7e7XbMoQlWPa331WcnDANBgkqhkiG9w0BAQsFADA2MTQwMgYDVQQDEytDQ01FIEcxIFRMUyBSU0EgMjA0OCBTSEEyNTYgMjA0OSBXVVMyIENBIDAxMB4XDTI2MDgxMTE3MzMyMloXDTI3MDIwNjIzMzMyMlowQDE-MDwGA1UEAxM1YXN5bmNvcGVyYXRpb25zaWduaW5nY2VydGlmaWNhdGUubWFuYWdlbWVudC5henVyZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeQHh5tmZ6ZYRlf_Q_PwPJqUN9t1MEgnet-7jMlsIvkZBhjMDVPNEFS1UtIM-3StxxWCffcghG31P1UcexhHelJCujRWSZYWfEIC7z_S8BBwxDhcc2peEIdNKD1O6UKvglkD8KRsPxRT-ZwQYY3qFNngixnx3EDhtWr1a9Kg46bdJJAIIzu3flYJzQzP3yVrFuRQn4iH75UvhgEByfjJaD75Ugw9JYkgO52nE5dVc2ddxoRMWzJKxp_V2Adp7VqTbMx2T8hO9PWqkltIPwoAXIubXFj3Z5A6Wj7hE7XmY_2cFJKJfspjiqwLyfnLqlv9pCLhIVdG31g8DAGZM7fc2lAgMBAAGjggSSMIIEjjCBnQYDVR0gBIGVMIGSMAwGCisGAQQBgjd7AQEwZgYKKwYBBAGCN3sCAjBYMFYGCCsGAQUFBwICMEoeSAAzADMAZQAwADEAOQAyADEALQA0AGQANgA0AC0ANABmADgAYwAtAGEAMAA1ADUALQA1AGIAZABhAGYAZgBkADUAZQAzADMAZDAMBgorBgEEAYI3ewMCMAwGCisGAQQBgjd7BAIwDAYDVR0TAQH_BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDgYDVR0PAQH_BAQDAgWgMB0GA1UdDgQWBBQbwB3CifQh36XI9i1lBe8503G1RTAfBgNVHSMEGDAWgBSs43L6A7Jznj2VyO-HW67dG6HtaDCCAbIGA1UdHwSCAakwggGlMGmgZ6BlhmNodHRwOi8vcHJpbWFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwa6BpoGeGZWh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3R1czIvY3Jscy9jY21ld2VzdHVzMnBraS9jY21ld2VzdHVzMmljYTAxLzY3L2N1cnJlbnQuY3JsMFqgWKBWhlRodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwb6BtoGuGaWh0dHA6Ly9jY21ld2VzdHVzMnBraS53ZXN0dXMyLnBraS5jb3JlLndpbmRvd3MubmV0L2NlcnRpZmljYXRlQXV0aG9yaXRpZXMvY2NtZXdlc3R1czJpY2EwMS82Ny9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jYWNlcnRzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWV3ZXN0dXMycGtpLndlc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQACHqTwaTHdHWv9uHzVkq0ltJxzjLPIQHJnP6ndMrFnY-TIxQDrABl7eSPlINC2qk1CFExKLmjYM8aDst83BXP-yH0AHIvBrOGVdZvqFE-8KPpkqE97J5NNLNr1lLkz4vWpbjmJhLvYmureoMJjtcLpQyz-EfACVM90Lkmu8ueIkp5mjzxa_-cMa3KYwbFXLO8huXYcrhLhvs31Z9czJKjlPptL0ZE1-rBlMXGqTdZUE1CfPRoTjMMRZe_jEJYNKyxhL_5fT2mLxdlYJsMDeTU5qM-x3lA74zxVrUvC0ZvCQdK629aZQUbWV74Uzk1DmAtUiDhGvnsOW-rug_uLr621\u0026s=2ED070N0ZTuw-f1HVDJ70D4TL-E7a_pmemS26JAcsDWdtZNt5piI-6t7rp4DAeUvveysVwPzvAf5ULh-g26b96AAV9pXZ4I1vo74l2qjDBTA1ho2Ia11r_3U1XAtiPoLjGufya8CYXOek1iwZu7oaFEZ-4uayFIei-jUkQWxZ9Jz8_HKfyzJZZA1qJtJWKMIu_-fMQRELNXCWOkBgzgrUj1MP39GcY4oa5lZoVmPJGhHGseHZ--ETS4Vun62lwfxl7XrPb2PN0L9g7S07Nil12FIWSbh_qVJ2DC5m-7fPlkYze7LvocmWvHPwMOLiIbvvBS9Ui4BX7syQlC25zGKXw\u0026h=siGPFN1sMGFTEzhJalRHQiDtbRbqB8eZGXQRBJUWkyo\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qhh4b47m8ncv/providers/Microsoft.Compute/virtualMachines/cidr-connectivity-test-test-vm/runCommands/e2e-runcommand-1789704441845701237?api-version=2024-03-01\": stream error: stream ID 33; CANCEL; received from peer)\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:293]: failed to create cluster role binding for external auth via VM\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d173e0\u003e: \n VM run command \"e2e-runcommand-1789704441845701237\" on VM \"cidr-connectivity-test-test-vm\" failed and cleanup also failed: Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/locations/CanadaCentral/operations/67651689-75ab-4f6c-aa6e-1746fbbdc2ad?p=c6bb2de4-260b-49a6-86a7-56c69bdc0403\u0026api-version=2024-03-01\u0026t=639253012427558463\u0026c=MIIHlDCCBnygAwIBAgIQTJW7e7XbMoQlWPa331WcnDANBgkqhkiG9w0BAQsFADA2MTQwMgYDVQQDEytDQ01FIEcxIFRMUyBSU0EgMjA0OCBTSEEyNTYgMjA0OSBXVVMyIENBIDAxMB4XDTI2MDgxMTE3MzMyMloXDTI3MDIwNjIzMzMyMlowQDE-MDwGA1UEAxM1YXN5bmNvcGVyYXRpb25zaWduaW5nY2VydGlmaWNhdGUubWFuYWdlbWVudC5henVyZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeQHh5tmZ6ZYRlf_Q_PwPJqUN9t1MEgnet-7jMlsIvkZBhjMDVPNEFS1UtIM-3StxxWCffcghG31P1UcexhHelJCujRWSZYWfEIC7z_S8BBwxDhcc2peEIdNKD1O6UKvglkD8KRsPxRT-ZwQYY3qFNngixnx3EDhtWr1a9Kg46bdJJAIIzu3flYJzQzP3yVrFuRQn4iH75UvhgEByfjJaD75Ugw9JYkgO52nE5dVc2ddxoRMWzJKxp_V2Adp7VqTbMx2T8hO9PWqkltIPwoAXIubXFj3Z5A6Wj7hE7XmY_2cFJKJfspjiqwLyfnLqlv9pCLhIVdG31g8DAGZM7fc2lAgMBAAGjggSSMIIEjjCBnQYDVR0gBIGVMIGSMAwGCisGAQQBgjd7AQEwZgYKKwYBBAGCN3sCAjBYMFYGCCsGAQUFBwICMEoeSAAzADMAZQAwADEAOQAyADEALQA0AGQANgA0AC0ANABmADgAYwAtAGEAMAA1ADUALQA1AGIAZABhAGYAZgBkADUAZQAzADMAZDAMBgorBgEEAYI3ewMCMAwGCisGAQQBgjd7BAIwDAYDVR0TAQH_BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDgYDVR0PAQH_BAQDAgWgMB0GA1UdDgQWBBQbwB3CifQh36XI9i1lBe8503G1RTAfBgNVHSMEGDAWgBSs43L6A7Jznj2VyO-HW67dG6HtaDCCAbIGA1UdHwSCAakwggGlMGmgZ6BlhmNodHRwOi8vcHJpbWFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwa6BpoGeGZWh0dHA6Ly9zZWNvbmRhcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3R1czIvY3Jscy9jY21ld2VzdHVzMnBraS9jY21ld2VzdHVzMmljYTAxLzY3L2N1cnJlbnQuY3JsMFqgWKBWhlRodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vd2VzdHVzMi9jcmxzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvNjcvY3VycmVudC5jcmwwb6BtoGuGaWh0dHA6Ly9jY21ld2VzdHVzMnBraS53ZXN0dXMyLnBraS5jb3JlLndpbmRvd3MubmV0L2NlcnRpZmljYXRlQXV0aG9yaXRpZXMvY2NtZXdlc3R1czJpY2EwMS82Ny9jdXJyZW50LmNybDCCAbcGCCsGAQUFBwEBBIIBqTCCAaUwbAYIKwYBBQUHMAKGYGh0dHA6Ly9wcmltYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBuBggrBgEFBQcwAoZiaHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdHVzMi9jYWNlcnRzL2NjbWV3ZXN0dXMycGtpL2NjbWV3ZXN0dXMyaWNhMDEvY2VydC5jZXIwXQYIKwYBBQUHMAKGUWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0dXMyL2NhY2VydHMvY2NtZXdlc3R1czJwa2kvY2NtZXdlc3R1czJpY2EwMS9jZXJ0LmNlcjBmBggrBgEFBQcwAoZaaHR0cDovL2NjbWV3ZXN0dXMycGtpLndlc3R1czIucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdHVzMmljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQACHqTwaTHdHWv9uHzVkq0ltJxzjLPIQHJnP6ndMrFnY-TIxQDrABl7eSPlINC2qk1CFExKLmjYM8aDst83BXP-yH0AHIvBrOGVdZvqFE-8KPpkqE97J5NNLNr1lLkz4vWpbjmJhLvYmureoMJjtcLpQyz-EfACVM90Lkmu8ueIkp5mjzxa_-cMa3KYwbFXLO8huXYcrhLhvs31Z9czJKjlPptL0ZE1-rBlMXGqTdZUE1CfPRoTjMMRZe_jEJYNKyxhL_5fT2mLxdlYJsMDeTU5qM-x3lA74zxVrUvC0ZvCQdK629aZQUbWV74Uzk1DmAtUiDhGvnsOW-rug_uLr621\u0026s=2ED070N0ZTuw-f1HVDJ70D4TL-E7a_pmemS26JAcsDWdtZNt5piI-6t7rp4DAeUvveysVwPzvAf5ULh-g26b96AAV9pXZ4I1vo74l2qjDBTA1ho2Ia11r_3U1XAtiPoLjGufya8CYXOek1iwZu7oaFEZ-4uayFIei-jUkQWxZ9Jz8_HKfyzJZZA1qJtJWKMIu_-fMQRELNXCWOkBgzgrUj1MP39GcY4oa5lZoVmPJGhHGseHZ--ETS4Vun62lwfxl7XrPb2PN0L9g7S07Nil12FIWSbh_qVJ2DC5m-7fPlkYze7LvocmWvHPwMOLiIbvvBS9Ui4BX7syQlC25zGKXw\u0026h=siGPFN1sMGFTEzhJalRHQiDtbRbqB8eZGXQRBJUWkyo\": stream error: stream ID 25; CANCEL; received from peer (cancel error: Delete \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qhh4b47m8ncv/providers/Microsoft.Compute/virtualMachines/cidr-connectivity-test-test-vm/runCommands/e2e-runcommand-1789704441845701237?api-version=2024-03-01\": stream error: stream ID 33; CANCEL; received from peer)\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100780897212043264","occurred_at":"2026-09-18T02:56:11Z","signature_id":"66f62a5d5d02e16dfffeec66474c962d8c14552e10473a54125784861158f0d5","pr_number":6898,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,2,2,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-19T00:04:52Z\n Ended: 2026-09-19T00:10:51Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/services-j8565632\", namespace=\"klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-19T00:04:52Z\n Ended: 2026-09-19T00:10:50Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8565632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.153:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8565632/providers/microsoft.monitor/accounts/services-j8565632\", namespace=\"klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-v6bbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2stnn2qj5n4300vnra0q4o64lcoi8994/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7012/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101080021098565632","occurred_at":"2026-09-18T22:44:47Z","signature_id":"a46151e80a37451e13b91d43941493c16950cf32997bf773ab888a8c5db06db6","pr_number":7012,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":1}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-18T18:04:42Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1089664-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/services-j1089664\", namespace=\"klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5 on cluster ci01-j1089664-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-18T18:05:44Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1089664-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", lease=\"governance-policy-framework\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/services-j1089664\", namespace=\"klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease governance-policy-framework in namespace klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5 on cluster ci01-j1089664-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-18T18:05:44Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1089664-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.54:8080\", job=\"kube-state-metrics\", lease=\"config-policy-controller\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1089664/providers/microsoft.monitor/accounts/services-j1089664\", namespace=\"klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5\", pod=\"arohcp-monitor-kube-state-metrics-6486647766-6vsrg\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease config-policy-controller in namespace klusterlet-2sti77pkq7agiit4e9p66ic3479hnrq5 on cluster ci01-j1089664-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100983315791089664","occurred_at":"2026-09-18T15:17:47Z","signature_id":"b2d9d84ce6e1c74ea14279f086b91ae4fc06d24e40b82a7ad2aa54e78ffa7d4b","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,4,2,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-18T07:08:32Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j0748928-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-9hl75\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-9hl75 (namespace arobit) on cluster ci01-j0748928-svc has averaged 125.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100815276730748928","occurred_at":"2026-09-18T05:12:47Z","signature_id":"476aed8123ee528db23f3663c1332194314ed537bec78310ea649964cd6c166f","pr_number":6157,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1ffc144947159d20ed7dd5010cb0a93b810641281a5d037d76b1920de5d6d537","failure_pattern":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later","search_query":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b28b70\u003e: \n clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b28b70\u003e: \n clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100747284332417024","occurred_at":"2026-09-18T00:42:37Z","signature_id":"82fd789c0354677fc94a578c2ebbcf48d32a60e2464d95dc9709afd5a7216752","pr_number":6972,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-p9htbjsmj226\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0031a8160\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-p9htbjsmj226\" to finish creating: GET https://rp.j6818048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\",\n \"name\": \"1616a745-fc49-408c-a84f-41679708fa49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:27:23.7095894Z\",\n \"endTime\": \"2026-09-18T09:28:03.46142651Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0031a8160\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-p9htbjsmj226\" to finish creating: GET https://rp.j6818048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1616a745-fc49-408c-a84f-41679708fa49\",\n \"name\": \"1616a745-fc49-408c-a84f-41679708fa49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T09:27:23.7095894Z\",\n \"endTime\": \"2026-09-18T09:28:03.46142651Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100863173186818048","occurred_at":"2026-09-18T08:23:07Z","signature_id":"1ff84f075d12052118fcca1c651ed549b6c5f0208069e0615f46732cae26b19e","pr_number":6900,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e86119606257db722a985501dec6db3353e9294ec584e08a2cf350342957ce78","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context canceled]","search_query":"failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: context canceled]","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n \"name\": \"c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:56:03.231032121Z\",\n \"endTime\": \"2026-09-18T03:56:41.678977419Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: context canceled]\nExpected\n \u003c[]error | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: GET https://rp.j2645248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n \"name\": \"c390c8d1-985a-452d-9ed6-80247abd9b5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:56:03.231032121Z\",\n \"endTime\": \"2026-09-18T03:56:41.678977419Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gpnvkzww4bm8\" to finish creating: context canceled]\nExpected\n \u003c[]error | len:2, cap:2\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100782327662645248","occurred_at":"2026-09-18T03:01:52Z","signature_id":"de20ef80e43125ed23aee61f6adcfb8942fa81561feaefb4e80cef19ed4db1c7","pr_number":6972,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"eaacc54ebd46cefe15abd433d5c3a394c2c9eab33016bf077e0076e05e8a3f60","failure_pattern":"failed to create route: the server is currently unable to handle the request","search_query":"failed to create route: the server is currently unable to handle the request","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-v458r6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001029700\u003e: \n failed to create route: the server is currently unable to handle the request\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-v458r6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001029700\u003e: \n failed to create route: the server is currently unable to handle the request\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7010/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023746776109056","occurred_at":"2026-09-18T19:01:10Z","signature_id":"c0b449ac08413f5f7083190089ef0d839099fa2699d64d1c9f2f8d26abbef529","pr_number":7010,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1a0fe5d7569a83b55b15427e5bd80f1ae31a9f5785e0f4d3715f223065550ec9","failure_pattern":"failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","occurrences":1}],"full_error_samples":["time=2026-09-18T21:55:47.666Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-18T21:55:48.312Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[armPermissionsManagerIdentity:map[certName:armHelperPoolCert-6 clientId:9c78a811-9c85-4eb0-9c84-d0843866c888 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpci01oidcj1086464.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpci01oidcj1086464.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] backupCadenceProfile:testing backupScheduleState:Enabled clusterScopedIdentitiesConfig:map[roleSetName:dev] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:780fc8ab-b71d-4dc8-a753-5e608208a59f databaseName:arohcpci01-rp-j1086464 databaseUrl:https://arohcpci01-rp-j1086464.documents.azure.com:443/ kubeApplierCosmosContainerName:Manifests-MC-1 location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-vh2hkbkg/pipeline@sha256:ec4f5311c601606f0bdcb4316ea9dff923b912c8e13a93addd5ac2c23dd9e86c replicas:2 resources:map[limits:map[memory:unlimited] requests:map[cpu:475m memory:1Gi]] zoneCount:3] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] logVerbosity:4 maestro:map[sourceEnvironmentIdentifier:arohcpci01] miMock:map[certName:msiMockPoolCert-12 clientId:50d052a6-7aec-48b9-b43e-b8692a8f0ead clientIdSecretName: principalId:a76148d4-cb11-4b93-9363-54ba8239b0b1 principalIdSecretName: tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-vh2hkbkg/pipeline:pull workloadIdentityClientId:09708096-ca85-4992-8eb5-667a9b9d3fce workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:780fc8ab-b71d-4dc8-a753-5e608208a59f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-09-18T21:55:48.312Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-18T21:55:48.698Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-09-18T21:55:48.964Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-09-18T21:55:48.964Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-18T21:55:49.102Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-18T22:00:55.361Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\"\ntime=2026-09-18T22:00:55.361Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 1/2\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101059271281086464","occurred_at":"2026-09-18T21:22:20Z","signature_id":"b8972dd0ff9a3ca0c20207306d152fa4afd8e4eb6a37b8d1c512cde87ef5ae3e","pr_number":6577,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"dac8a1b25a3188542fe843437616ad526549332051ce7885378b1b60aa94182a","failure_pattern":"failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kube-events (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.1904761904761905,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kube-events (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-09-18T20:01:01.932Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 description=\"Step kube-events\\n Kind: Helm\\n\"\ntime=2026-09-18T20:01:02.379Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 values=\"map[enabled:true image:map[digest:sha256:5177425b194b5d289664c62bedbdc95a42605bbfcde48fc7ddbb31bee0194b90 registry:kubernetesshared.azurecr.io repository:shared/kube-events] resources:map[limits:map[memory:unlimited] requests:map[cpu:200m memory:512Mi]]]\"\ntime=2026-09-18T20:01:02.379Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1\ntime=2026-09-18T20:01:02.574Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 namespace=monitoring\ntime=2026-09-18T20:01:02.761Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 dryRun=false\ntime=2026-09-18T20:01:02.761Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1\ntime=2026-09-18T20:01:02.827Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1\ntime=2026-09-18T20:06:05.062Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 err=\"resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-09-18T20:06:05.062Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kube-events stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101023754950807552","occurred_at":"2026-09-18T19:01:12Z","signature_id":"30e287a059c7694cef227767592b23353845f4281b98b8c7ff98febc06270f17","pr_number":7002,"after_last_push_of_merged_pr":true}]}]},{"environment":"int","summary":{"total_runs":3,"failed_runs":0,"raw_occurrences":0,"matched_occurrences":0,"runs_affected":0},"rows":null},{"environment":"prod","summary":{"total_runs":15,"failed_runs":9,"raw_occurrences":78,"matched_occurrences":78,"runs_affected":9},"rows":[{"environment":"prod","failure_pattern_id":"5ca70ffb1d249122818522a39554733558b9485ca9268cbed2aebd2e521248e7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster placement did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":51,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.666666666666667,"anchor_occurrences":51,"after_last_push_count":51,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,51],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-n6q68g with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000be8cc0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-n6q68g: failed waiting for cluster=\"np-version-upgrade-cluster-n6q68g\" in resourcegroup=\"rg-np-version-upgrade-n6q68g-z5wvsgjdr7l9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n \"name\": \"47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:24.695867176Z\",\n \"endTime\": \"2026-09-18T20:24:27.02816156Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-n6q68g with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000be8cc0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-n6q68g: failed waiting for cluster=\"np-version-upgrade-cluster-n6q68g\" in resourcegroup=\"rg-np-version-upgrade-n6q68g-z5wvsgjdr7l9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n \"name\": \"47d83203-d5e7-4af0-adbd-1321b73b5d96\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:55:24.695867176Z\",\n \"endTime\": \"2026-09-18T20:24:27.02816156Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000de50e0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-bddwvtnwz8x8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n \"name\": \"1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:54:45.082839017Z\",\n \"endTime\": \"2026-09-18T20:23:47.024709747Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000de50e0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-bddwvtnwz8x8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n \"name\": \"1de5df02-d5bc-4ea1-8f35-f119420c215b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T19:54:45.082839017Z\",\n \"endTime\": \"2026-09-18T20:23:47.024709747Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010085a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-266qhn49mfp5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n \"name\": \"724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:21.25552087Z\",\n \"endTime\": \"2026-09-18T18:35:23.923631594Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010085a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-266qhn49mfp5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n \"name\": \"724dbb1d-6cfa-4cff-8bf4-5e1e742772df\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:21.25552087Z\",\n \"endTime\": \"2026-09-18T18:35:23.923631594Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster placement did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"020177279fd83a306292e61e3b6eb559ce1bd5ba1e079600d6fe28da054a781b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"08737fbd1b615be947880829b89bc4c3a56995f666d4512eff62626a308dd556","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"087e1145946b4175b23e226d2c422f8fbe6adcface041612342c65176f135291","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"12c88ab455c66941eb056387e08ab482c4399c448c1844b1db1d64ca2a21923b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"18459d5e026c91320fea1e6f4619d65e0f3c2cce677a7bf1890dbe9529aa82dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"18cde222ab56ffb759e053d76ee6e19c739c516c2db2d311138d345c27569034","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"19596332cc8edb7c2fe7ff05b62421eeeea53668cb594d6f172e695930cfc93e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"23eff0283d14160643a2c1b951b106f62c524eb771d25036e9a90406a0924c53","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"279014bba3c91c8c1147706aa208a7ca8dab431b40a05ae4dc2507ffe4bd82d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"2a4e0071788107d1b0f28cafcae777b74d59c2ed92ca22e3caeb7564b1685be1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"2bd526e04864e297297108f31c386e629877109b67a0cd5577f66e966f4820a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"2c5047fbc800377676e3159b01faa4b14e97abaaef72bb1d0f7c177a6a8e963a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"2e5039616cb1c8d7f4ac14c50b2a06f1fe40081131c5acd543f28d2cc514abec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"300283533309312abfaf8eb06c5bc243249dba952ac99795631a8cca96c1b04d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"3a36b2cb0e45db6b1b5adf537bf6a65a0dc2daa7ebc74fcf104789a0636dc715","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"4d196dbdd6adc5cbd0b42174989b35c61699ec802a9b43e61050ca43a4a526db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"55df98f2e805e8d535825529f230ec77e5caf64b117f366392092a2ef0f04238","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"55f11975675616706103b6b7daff6a4b423e0747ce59c5e7da66f3dc349981e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"596b0ffa1cf24f21e4c89da0dd1bf8032daa39c9dfe890cbc997391f5e16d758","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"5bd085aa7d4e7efc364c83356bec393db7595ff945fba164d18f342fedc7d074","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"612c1b0bfa8c63e918534f55b46917bda5292bb668f43f5109397135c5f16839","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"7d0f6a2cda1c21acb4480d38818269f535fbffb2494dac6f98b849200db6bc5d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"807a30302c1afe55628a0e91c1d8d2d1cf846d02300aeafd38841fdc5629959a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"88f393269cef98e4be13d3c8ab69c7daf8948bb76a2277ef306df923eea104ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"8c17a49c3b1087a5252885b16d63b8887ba96a24c838b70dae74c83a59595562","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"8e3503c7dac56f3fd400f32d8b3f5b71ecf5acd0c55bc2d5e52b0b130348d093","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"8f8a4a3e3e075b660c1fb0534694eabcbec0ee97ddf1c7de50ae027a7c989e5e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"9253c3493335b5dfd398622bbe3451ad8bddf74417d1a8c0037aece178838cde","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"980439aafe10d6730454288b03e5c4b325ad4d9a23209ccce9ea0925ca8727b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"99a0fc3be3299d125a1312f83223671d90940dffaa62fee2e5b18166cfca67f4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"9da2d69669521624ba370e29cdccd478000fd77fa5acf531ed0100da09e598f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"a086d8d16ebbda1f55085340c84766813ece581729f4e4db8f61f9c0b1339c8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"aba4ff8b5edea60411faee257f0fcca1514022f58c987167a980974659ac599f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"ac027a2fae7f9c119c5d13e433aca363beeb537c7f13236bce5b4509168d72e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"b09c9c83b49154876ea9edaee019cac3ced0de0e27ddbe6889ee50f6b423f2ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"b971663cd5f630a57997cfad6d7efdb8293b9c2c40b6d6ab7621acde7a5c5511","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"bec4f85eb91db9f106f1504bbcfbd5d619f318abcff9dccca5f00cd3af48e378","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"bfbf541997c9dcf2457e39f6d9a3f3675b64df774c7c81e642e24bee8b14665a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"c71f8c34107674e9b83007fd4c4382acbf933644d3d70e04b10c2bce3d213437","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"c9577e8c1b375d41d27d0e18df18642cc9e2a302455c27112af6cdf04b27da87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"cb28d0b51212f011a6a9a6a3530556dc58c4a4d691d64d362e422cfc091e42ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"ccebff601285f1ea3ac04ae7a355282a8c0e14e4f8eb582eebc840c11f6d9a08","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"cf464e2390b41d88251c211ae76f31739afd822d6d94414c2b9fbfc2f709feaa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"d992a344dec95549926180b213136a61987bad3b2b2cfe5a8cedabfb0cba64d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"d9b87f83858caf3b228ef9b132bcd625c28601b6944272225765d7e4212bfbd4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"eabc0e9f060f2e7a3e41ea442762aca89d3441bef1655db8dca9669defa6ff8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"ec2f8559597c2e2e96353159ad3c18cb0e7f4c1ae2d3f60c226c0178a7ef5d2b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"ed3aaafec71e925920ce96600e6accd52368d8c067bd7c98fb0f3dd54c8990b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"f52d9598c009df2683fa8e80fa13ffac13c236506f242fa3de8742e288cc9518","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"f6a9a5ad677f486ebe2e2116b48d59eff2dab4dc02f9785191be923be7559549","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"fcaef2f74196ab4f30c8d92bee5653f6add69a944685179ec76976f1aebee54b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","failure_pattern":"ERROR CODE: SubscriptionNotRegistered; provider Microsoft.Insights","search_query":"ERROR CODE: SubscriptionNotRegistered","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":33.333333333333336,"anchor_occurrences":5,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,1,5],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a45890\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a45890\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-d9mbtgrlrxw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00117a2a0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00117a2a0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-986dm5j578hd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00088e570\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00088e570\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-92rgrk84fv8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100894117604102144","occurred_at":"2026-09-18T10:26:04Z","signature_id":"3269dcb6be86d6a6047ecae085ca193c45aec85e3774519113b1593c115dfa87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100892735677403136","occurred_at":"2026-09-18T10:20:35Z","signature_id":"d7f2aa1ec90b6da0508d97952c26f5e68a44fec6becea93e11a0c8acd2ecf80a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100890223054426112","occurred_at":"2026-09-18T10:10:35Z","signature_id":"5533b31e4276757a4afc74d1ad67148e45245ccef8ea260d65b9e08077693a1d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","signature_id":"6fd32498c414e944f0c010bd303fd297e25f438aa938164d7b89331e4fe1bf3e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100762866352132096","occurred_at":"2026-09-18T01:44:31Z","signature_id":"f164ce20210c5901433b68d8b32c7bafd9db9fbdb4b9f088ddfe4a53eb8ff36e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":5,"runs_affected":3,"failed_runs":3,"run_impact_percent":20,"anchor_occurrences":5,"after_last_push_count":11,"also_in":["DEV","STG"],"trend_counts":[0,0,0,0,1,5,5],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002d105c0\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-wbftvl6pxn6v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f5be950-3d76-400d-bba8-80953e9b85fb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f5be950-3d76-400d-bba8-80953e9b85fb\",\n \"name\": \"9f5be950-3d76-400d-bba8-80953e9b85fb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:41:11.725184381Z\",\n \"endTime\": \"2026-09-18T08:00:14.656645268Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002d105c0\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-wbftvl6pxn6v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f5be950-3d76-400d-bba8-80953e9b85fb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f5be950-3d76-400d-bba8-80953e9b85fb\",\n \"name\": \"9f5be950-3d76-400d-bba8-80953e9b85fb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T07:41:11.725184381Z\",\n \"endTime\": \"2026-09-18T08:00:14.656645268Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.32\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efdee0\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-dd2djv\" in resourcegroup=\"rg-np-version-upgrade-dd2djv-f5pt9zl6csx7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fcd870-f035-4d43-a07e-056005a0cd12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fcd870-f035-4d43-a07e-056005a0cd12\",\n \"name\": \"34fcd870-f035-4d43-a07e-056005a0cd12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:23:36.136458591Z\",\n \"endTime\": \"2026-09-18T03:42:38.777220989Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.32\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efdee0\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-dd2djv\" in resourcegroup=\"rg-np-version-upgrade-dd2djv-f5pt9zl6csx7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fcd870-f035-4d43-a07e-056005a0cd12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fcd870-f035-4d43-a07e-056005a0cd12\",\n \"name\": \"34fcd870-f035-4d43-a07e-056005a0cd12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T03:23:36.136458591Z\",\n \"endTime\": \"2026-09-18T03:42:38.777220989Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001092648\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-tgqlbhrjcjzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4e2d962-cb10-4c5d-b5e9-88c9efab7468\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4e2d962-cb10-4c5d-b5e9-88c9efab7468\",\n \"name\": \"f4e2d962-cb10-4c5d-b5e9-88c9efab7468\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:15:45.572313063Z\",\n \"endTime\": \"2026-09-18T02:34:50.003063171Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001092648\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-tgqlbhrjcjzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4e2d962-cb10-4c5d-b5e9-88c9efab7468\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4e2d962-cb10-4c5d-b5e9-88c9efab7468\",\n \"name\": \"f4e2d962-cb10-4c5d-b5e9-88c9efab7468\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:15:45.572313063Z\",\n \"endTime\": \"2026-09-18T02:34:50.003063171Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100826864959361024","occurred_at":"2026-09-18T05:58:50Z","signature_id":"7cb7501ef4b819fdc17f5b5e3ca2619d486971f5a3c25a5533b8ff273a412172","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","signature_id":"60cc795ca9532d84d0a6a8c864d8b05a2f5285c1bfd6c8aa612a88d8890d8bb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","signature_id":"6e4745afece375c35c57231f904892210b15b74988a6f2baceda810fa43da8e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100766832733458432","occurred_at":"2026-09-18T02:00:17Z","signature_id":"8be7c23a1e56f277787ec21fa08eb46e46af4b8580d42385a8b71372839091d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100762866352132096","occurred_at":"2026-09-18T01:44:31Z","signature_id":"812ff7352462ae5b2ffc2a98177a772e72a7a199983bda3af2609cfd08e8cb77","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.666666666666667,"anchor_occurrences":5,"after_last_push_count":14,"trend_counts":[0,0,0,0,5,4,5],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f5e360\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-cx8sqpwt9s45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f5e360\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-cx8sqpwt9s45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002bb68a0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-t78vdzzqpblp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002bb68a0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-t78vdzzqpblp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002804f90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-grcsb5ww9xqw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002804f90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-grcsb5ww9xqw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4e7909db88252003612c7eb675142841e8737860b4601bc7ba1c1c1147d59585","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"6b756bbe3882dd3ad1f64906dbb7701108fcd8092065edc0fdfc339e155785c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"9ef798cef3d572a0e66143f21817eeeceae70fa379c7c12f948b17be196688fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"a1f4334212b041c83192bb8549659dd10bbe7f9e8bd239e3cb44f42460834cd6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"d656f1256e1df9e299a9ee99092b293d1d1b5f5763759abb016ae0c94e36cbee","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.666666666666667,"anchor_occurrences":4,"after_last_push_count":8,"trend_counts":[0,0,0,0,0,4,4],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011be60\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-tcgt6v\" in resourcegroup=\"rg-np-skip-minor-tcgt6v-cwclsxtppk9g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-tcgt6v-cwclsxtppk9g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-tcgt6v/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011be60\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-tcgt6v\" in resourcegroup=\"rg-np-skip-minor-tcgt6v-cwclsxtppk9g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-tcgt6v-cwclsxtppk9g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-tcgt6v/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013480c0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-mpcclt\" resourcegroup=\"rg-np-dg-minor-mpcclt-kpfflj2pg95l\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-mpcclt-kpfflj2pg95l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-mpcclt/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013480c0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-mpcclt\" resourcegroup=\"rg-np-dg-minor-mpcclt-kpfflj2pg95l\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-mpcclt-kpfflj2pg95l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-mpcclt/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00160a220\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ddwqfs\" in resourcegroup=\"rg-np-version-upgrade-ddwqfs-m7xcgsn6lcv6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ddwqfs-m7xcgsn6lcv6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ddwqfs/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00160a220\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ddwqfs\" in resourcegroup=\"rg-np-version-upgrade-ddwqfs-m7xcgsn6lcv6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ddwqfs-m7xcgsn6lcv6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ddwqfs/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"491628b312668f06a0a9bf2464e7de82cea05fd0fafcb43e52e269cbc904606b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"8807298fe1e8269d8ba1826e326fe8888ecc37b86d4680f87d9ef650960b1b45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"889f4b2cac9cd859f4a4017cf24ccfb20aa30ab6ad074847bdec286491246277","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"b07971657c34858950043966384bf3450b2bd8793b71995b336ae5e99ce5d756","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"e3e491019ecf3f140e87c882b7d45ae9d3071ed1df24cc639d33c7a9d0e29087","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":13.333333333333334,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014ced60\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-wvt6tvvldjdg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n \"name\": \"80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:15.268087105Z\",\n \"endTime\": \"2026-09-18T18:35:15.027123676Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014ced60\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-wvt6tvvldjdg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n \"name\": \"80eaa57a-6c21-4104-9379-5f7032bfe26e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T18:06:15.268087105Z\",\n \"endTime\": \"2026-09-18T18:35:15.027123676Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000860220\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9vjp5gmczk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n \"name\": \"df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:08:07.344249043Z\",\n \"endTime\": \"2026-09-18T02:37:10.01628355Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000860220\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9vjp5gmczk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n \"name\": \"df01aa2b-a5c5-4899-b645-5ce58f5f8cee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:08:07.344249043Z\",\n \"endTime\": \"2026-09-18T02:37:10.01628355Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003701249380352","occurred_at":"2026-09-18T17:06:01Z","signature_id":"e22784c3104fac10937a42e7b5069677dd3f893046b62e795e223bfab93b9fcd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100762866352132096","occurred_at":"2026-09-18T01:44:31Z","signature_id":"736f2b0814fabfc59b7e462d9755a0e748e4d396c2743f81540491fd582dab16","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.666666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.666666666666667,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,1,1,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00298e7c0\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-9f9tbz46mct7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-9f9tbz46mct7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00298e7c0\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-9f9tbz46mct7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-9f9tbz46mct7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"19958c8a5897194ef3182c6ab16be4ac9029906aade55bb4256e8a9bbd189d3c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.666666666666667,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,1,1,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00055c870\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00055c870\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"31f227f2999d078a72461d40e7b0f7d39a61c55e78688c72750e61796cea0c0e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.666666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-tzdswnvvvrrb\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.666666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001376700\u003e: \n failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-tzdswnvvvrrb\" to finish creating: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001376700\u003e: \n failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-tzdswnvvvrrb\" to finish creating: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101083983914733568","occurred_at":"2026-09-18T23:00:32Z","signature_id":"e69d26f721f1d960b9d056d864133ef61ba83a27918de693c75cfc743d5fc4a8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ql4cwwsm948w\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-ql4cwwsm948w, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.666666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0010a86c0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ql4cwwsm948w\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-ql4cwwsm948w, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0010a86c0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ql4cwwsm948w\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-ql4cwwsm948w, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2101003476451463168","occurred_at":"2026-09-18T17:02:57Z","signature_id":"60ea4497c6186b3e803d7e6ac044255c8c915fd98b1ff264e2058f0d46b0df0e","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":2,"failed_runs":2,"raw_occurrences":54,"matched_occurrences":54,"runs_affected":2},"rows":[{"environment":"stg","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":52,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":52,"after_last_push_count":52,"trend_counts":[0,0,0,0,0,0,52],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a102e0\u003e: \n failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-5t8m268g2ktr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-5t8m268g2ktr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a102e0\u003e: \n failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-5t8m268g2ktr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-5t8m268g2ktr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008ac220\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gp6ml4ltbxq9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-gp6ml4ltbxq9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008ac220\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gp6ml4ltbxq9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-gp6ml4ltbxq9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bba1c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xm5czf8ql9ll\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-xm5czf8ql9ll/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bba1c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xm5czf8ql9ll\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-xm5czf8ql9ll/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"085c6afa73df8e5dcbb8a9e0c59a48a53cb276afb92a752caf002a49c403af63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"0e7c33b837c87670cd35a11641aefa3118943334f501baf884bc7c01141848c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"0f82bd0c9700cf4a07c13d1f48415f495ffac7b49c2a7ae3841ea9c8faf79ca5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"10f37d08fe2d36b0048e252aedb2d548e4ca8db071149c417d5ce17b13fa36d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"1153df3f24dc8bcdeecfdd99c8141522fe1f0d9dd14f566bee678a30941ac49f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"1dd5e0dc7e15d356bc9a171360fd68f57e762e2e53e6cea924436fb68b6fb102","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"252c4afaeebb14c4115f0be666abd104e7adc80f40991bcc6beaa3ff85c91e84","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"25dbf965d6f20a1a7cec9776ec1e87245a3e50198ba091a7741329a88e0da378","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"265ce4c4fa80432d1672154f22571f629b7a60e7c351781f21d111b2d1880927","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"2740332bfcda380c3c02e8ee1f792b3568b04678a77023d1486f19de701aeb61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"27e6130cf36baa158d6711ebe28c73dffa820785f17bbc686ff8fc7d38fbdb8b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"340421cbdb7899927b7ac5fc7d0aa57d005e6eb5d784359ad31a36aed34461d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"3f18ddd10272127c408a7d9bde4eddaaa27e6f085f80cf7d94ab9be21a380754","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"42f7a4e6cab05c380f4d5d815367e49f29a7e4b22dcb36d855a4a05388b7e8cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4455b438d122ea70538db65aa4dcf60523e82a18567f93d3872e7891ddeb2d9f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"44de21e589e428af479c9ccfb6a72f778841ee48c0b271fb47903606d3baee24","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"483e21afcf9632ea6968f51c68da44c7e6f45d9a5418bb6f513b2bec6b1bbce2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4ba5ad9b16c83f433ab867cff2637f8bd36aa451460f9d15a478f5eb1eda9a65","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4c2e2bc46dbcb65a27b428b8f5c2170eb0f11ba205842dd52e99ae5bde4b31fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4d36c1670f69a8378c623f46ed5e24f3787ec0a5de39295ced2ebfe81f29cefa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"4f0f437ca045765d2d2b24079491f21e1fd5a85db0b44958c8034b2e66199908","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"562b8f83f84dba1886b3065181ffee89aab2e6673b7333d41f76aef2eaad1f4e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"58f07f406d642a91c9bddbe8fdd7ede440f08a2dd03a6f5a5c819b498466cf11","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"5e002cf10865e514b0815eed29286f8da9237e5aa8330589afaeef395b289efb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"677da6ac04d265e57cafd47671de87d4257ef7d06221d84329ebe1ab461fca66","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"6adb067c216551bfe29f1e3b77d6113e9d3d513c52d7c6b6659f174eed5dbd71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"71d45e9348cce618a98ba3a9cedd12637edd7a5d0db05784f92b3cf08108bd98","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"71e62b511a0fe354af7045a804df32da837a8d8a1285d20120469dc2b9b474a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"8b5bf6763af34e2d91fc69a112089a080b1da4da152d93af2ebb7174e191d48d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"97623f5d3f0d0fd367349a611b76873782d9adb09766bc94fe24252159b1769d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"998b87f452d353eeb973a256989175c16ceae9088c3fe30ce72eb5beea463849","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"a4bd95380032fc5885c8b2fd10eb51f3df30c29355f6b414b5b33fef1056e9a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"a9d975f128c27c084e7e5fd23daa85c45a8d628e9193f529805c3f218b289677","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"b44a4a5abf9df504b29cd303adc2b536d491938b5012cb47d37469bc577abd3d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"b89878daa0c4d5d3cb3d93a46690c3d9ddf063236bf34cd2f69bcdf3bb5f245a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"bc25c842a1235045614d605acc90fea01e39dbaf7d68ff406505237e93f1e16d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"c0258f3c8ab984584c9b6314be2ccb30e5ffa2b206f4ab774c595c07d6b813f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"c3410d264cf0132fe4f17940cdf2ec821ec784b23b0faff95c971f7658965387","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"c68022f2f40c275eb8c23a59749c7f7e4af0d44ada9136f9192d94b24e86254b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"cb988d58c89113961f03df82043150415e9b6ed049bf1f878c0c0b5c9eb97486","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"ce5b35b01c844ca412a4cc92791058f44a790b6ff65cc19fb352ae11b15d2ae3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"cf3269165eebc8b281476a0026c2b3449c115912f9f0f7f746f7524bfed7b58f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"d66e9a51e6356e3da95cc4251424afb6ce526cd142562c82ca1f05f8b3157c1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"db766735f9e579f4552ed8a3c29b312be7add829d329efde1debee620ae19b46","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"e1a746b36f9a4f74e169ab5b044f8ffbb4e94a6ac47cce6553d8ffee67ae68fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"e5ac7068f8a4f91ce6b642a6a37bd31dd8ace822a1805ae8325512e343c782e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"e94453651233184d87c4057afb3b9acb271e36b21383edb286efa6f234f89bef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"f0973137271b8ecc421edadd409b7fe63accbff0882cef646035b79a0aa8c22e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"f0cc45ea9fca99b68f2c8c623264abfb0139dd3f1bc78462db9705e9cff98ef3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"f6f1275a16afdec341158ade8fbc4ce79c93972342a64ac61c74f29d86d94b33","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"f7953744450082a3ec5bc94fd5cc7b0b148b3213f668f2afb80a43e6f0953ba9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101083983956676608","occurred_at":"2026-09-18T23:00:32Z","signature_id":"ffc3ee48d637c5b8ce881fb48a931d89a1627b9e6bc94e2411754ee778248a76","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":5,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,0,1,3,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-h29zrk\" for cluster \"cluster-candidate-4-21-h29zrk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009df3c0\u003e: \n failed to create NodePool np-h29zrk: failed waiting for nodepool=\"np-h29zrk\" for cluster \"cluster-candidate-4-21-h29zrk\" in resourcegroup=\"rg-candidate-4-21-k7gt4s25kh5h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eba02645-cd7e-4330-9e90-bb7a800d75d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eba02645-cd7e-4330-9e90-bb7a800d75d5\",\n \"name\": \"eba02645-cd7e-4330-9e90-bb7a800d75d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:38:03.543368376Z\",\n \"endTime\": \"2026-09-18T02:57:09.03735448Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-h29zrk\" for cluster \"cluster-candidate-4-21-h29zrk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009df3c0\u003e: \n failed to create NodePool np-h29zrk: failed waiting for nodepool=\"np-h29zrk\" for cluster \"cluster-candidate-4-21-h29zrk\" in resourcegroup=\"rg-candidate-4-21-k7gt4s25kh5h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eba02645-cd7e-4330-9e90-bb7a800d75d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eba02645-cd7e-4330-9e90-bb7a800d75d5\",\n \"name\": \"eba02645-cd7e-4330-9e90-bb7a800d75d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T02:38:03.543368376Z\",\n \"endTime\": \"2026-09-18T02:57:09.03735448Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2100766832767012864","occurred_at":"2026-09-18T02:00:17Z","signature_id":"16ebc00197f5d484a478287dbcf774e95611e6748652e8b6528ea5b7d11de1a1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"ab5d80437500581ad7ce3942572aca8a7c5ea9eda26f4367215f73be1501d7a0","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"npupgrade-4-21\" in cluster=\"np-version-upgrade-cluster-nl8qnf\" resourcegroup=\"rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-21 in cluster np-version-upgrade-cluster-nl8qnf in resource group rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-12..2026-09-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002292870\u003e: \n failed waiting for nodepool=\"npupgrade-4-21\" in cluster=\"np-version-upgrade-cluster-nl8qnf\" resourcegroup=\"rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-21 in cluster np-version-upgrade-cluster-nl8qnf in resource group rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002292870\u003e: \n failed waiting for nodepool=\"npupgrade-4-21\" in cluster=\"np-version-upgrade-cluster-nl8qnf\" resourcegroup=\"rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-21 in cluster np-version-upgrade-cluster-nl8qnf in resource group rg-np-version-upgrade-nl8qnf-jssxs5wlfz7w, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2100766832767012864","occurred_at":"2026-09-18T02:00:17Z","signature_id":"ea0f58ce05963824cd5712a4a308420855630745ad3e354951e106ba980a6661","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}