{"meta":{"start_date":"2026-08-19","end_date":"2026-08-25","start_at":"2026-08-19T00:00:00Z","end_at":"2026-08-26T00:00:00Z","timezone":"UTC","generated_at":"2026-08-25T21:11:41Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":634,"failed_runs":509,"raw_occurrences":1766,"matched_occurrences":1667,"runs_affected":410},"rows":[{"environment":"dev","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":116,"runs_affected":59,"failed_runs":59,"run_impact_percent":9.305993690851736,"anchor_occurrences":116,"after_last_push_count":20,"also_in":["INT","PROD","STG"],"trend_counts":[18,2,3,0,4,0,31,76],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":85,"prior_last_seen_at":"2026-08-22T18:55:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":9},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":11},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":8},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":7},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":7},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":14},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":12},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":10},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001af61c0\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-q5hktmktf7r4\" to finish creating: GET https://rp.j6821248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/656a6343-5635-46ba-ab12-bb1c8bf913ef\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/656a6343-5635-46ba-ab12-bb1c8bf913ef\",\n      \"name\": \"656a6343-5635-46ba-ab12-bb1c8bf913ef\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:31.574727642Z\",\n      \"endTime\": \"2026-08-25T19:36:32.549404211Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001af61c0\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-q5hktmktf7r4\" to finish creating: GET https://rp.j6821248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/656a6343-5635-46ba-ab12-bb1c8bf913ef\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/656a6343-5635-46ba-ab12-bb1c8bf913ef\",\n      \"name\": \"656a6343-5635-46ba-ab12-bb1c8bf913ef\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:31.574727642Z\",\n      \"endTime\": \"2026-08-25T19:36:32.549404211Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-cdmspj5xt8hg/cluster-candidate-4-20-t5rwd6 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d8c2c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-t5rwd6: failed waiting for cluster=\"cluster-candidate-4-20-t5rwd6\" in resourcegroup=\"rg-candidate-4-20-cdmspj5xt8hg\" to finish creating: GET https://rp.j6821248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dc6c49d-a480-49d7-a43d-d68d7bc97bfa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dc6c49d-a480-49d7-a43d-d68d7bc97bfa\",\n      \"name\": \"5dc6c49d-a480-49d7-a43d-d68d7bc97bfa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:20.683071275Z\",\n      \"endTime\": \"2026-08-25T19:36:22.56632343Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-cdmspj5xt8hg/cluster-candidate-4-20-t5rwd6 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d8c2c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-t5rwd6: failed waiting for cluster=\"cluster-candidate-4-20-t5rwd6\" in resourcegroup=\"rg-candidate-4-20-cdmspj5xt8hg\" to finish creating: GET https://rp.j6821248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dc6c49d-a480-49d7-a43d-d68d7bc97bfa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dc6c49d-a480-49d7-a43d-d68d7bc97bfa\",\n      \"name\": \"5dc6c49d-a480-49d7-a43d-d68d7bc97bfa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:20.683071275Z\",\n      \"endTime\": \"2026-08-25T19:36:22.56632343Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ebb040\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vscttbdw75nd\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/818a62a8-8d93-459d-8682-cfab29d0efcb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/818a62a8-8d93-459d-8682-cfab29d0efcb\",\n      \"name\": \"818a62a8-8d93-459d-8682-cfab29d0efcb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:19.815809316Z\",\n      \"endTime\": \"2026-08-25T19:39:23.985097515Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ebb040\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vscttbdw75nd\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/818a62a8-8d93-459d-8682-cfab29d0efcb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/818a62a8-8d93-459d-8682-cfab29d0efcb\",\n      \"name\": \"818a62a8-8d93-459d-8682-cfab29d0efcb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:19.815809316Z\",\n      \"endTime\": \"2026-08-25T19:39:23.985097515Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314723616821248","occurred_at":"2026-08-25T18:14:37Z","signature_id":"0076fbe063547b07110db9c853686695f6d9b45408e17004b5c20f9f45967910","pr_number":6682,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314723616821248","occurred_at":"2026-08-25T18:14:37Z","signature_id":"77c50bc17b9d53d03c763973e2fb5dbf388cfdf6e7d1846e1e64d2e0d6d3a12d","pr_number":6682,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"58bf9ab1eb1b504ee63dc64568b36cd225e7db6ef55de533f9e062770f4ac7f7","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"d9c26ef58d5fcf8c49cf5a4e45b95fc06f0f9d202df046f2ea975e15fe32ba90","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"f72cda8e54add5e222bb916eb78bd6882ff8c549adafd27df57b6813b5fa6eee","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"f731da949e2cbc194b5598178bee5be2ece9258202ff2948a41a7f722a5d15b6","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311205338157056","occurred_at":"2026-08-25T18:00:08Z","signature_id":"08d84795c86bdd39c92b3ef7ed4171353ac5d7a95629fb6767e9d5e53dab0575","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311205338157056","occurred_at":"2026-08-25T18:00:08Z","signature_id":"09f8cbe70cbae7527497ee988bb4e3ca9bd72951deeb0cf5d80d73cf512c0032","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311205338157056","occurred_at":"2026-08-25T18:00:08Z","signature_id":"87d15fc3b3d51f2f525f669b15efa9810a7a54c78626d39624a2206eb8b40682","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311205338157056","occurred_at":"2026-08-25T18:00:08Z","signature_id":"f7690a17f140753a646175aaf07b86c6525354b0ca8e94c3547454592f5e4408","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311205338157056","occurred_at":"2026-08-25T18:00:08Z","signature_id":"fc8cdd9240844df1f37ddc400bdd2c5a950866210e7f77cce8d91f909ab5d0c9","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092295016859832320","occurred_at":"2026-08-25T16:56:19Z","signature_id":"0669947174a86499d5670bc164ea47f8372430e063dda27921fe41db635f2d1a","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092295016859832320","occurred_at":"2026-08-25T16:56:19Z","signature_id":"ca1da36a392a93e6f3a6490b4497374f23e1bdcba26c42be24847ece14ed3f61","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284063539793920","occurred_at":"2026-08-25T16:12:48Z","signature_id":"b168e3a1107e3e1902272b9ff4556d75fd8dfe44f624dfd2607bc5f72bb938f0","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284063539793920","occurred_at":"2026-08-25T16:12:48Z","signature_id":"f71dae65ad1aafde8d4f6ae00fb97c69b8fec2ec7c262dff3ab35decf7d7de86","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284042668937216","occurred_at":"2026-08-25T16:12:43Z","signature_id":"81a27532541d67e6137f3b18dfa24f47172a110e72475bc734463343f3f6e146","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284042668937216","occurred_at":"2026-08-25T16:12:43Z","signature_id":"87fa0e9ed440b5075b8c246f9d4ac0dfb3a65d6379a21ec17eac51d7fe33d902","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092283360171790336","occurred_at":"2026-08-25T16:10:00Z","signature_id":"e091e7c3963b1d439bcf214121c57fa63b1447fa3dcc926afb3e313210a13c5e","pr_number":6630,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"2d44f9e1ef87d22b647e496ad891393923d8b6071781ac4f3325409453b16aec","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"431183966ffec9d316eabffd849e7f6e0b0f56934e37ad7b18ebee3f9d0c7785","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"6d0a93471cca29b596f2665a2e4b14d8f14b38ab392d8cc47699b7365343a0d3","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"a34616cf01ee0e699438c4845004c7e68738e05288f74d735ab18aa5dbd40940","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"aa50862b2be22c2ff41532ef6198e1092baabb54df9335776d5e11eaa6a79bd4","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"4f01337717137a4b91f0a8939b1c66d8d9f1d513029bbfdaa010a32c379be1fb","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"9a5de514cbd74ec8858c418a7e3bc6208cf5c8a2a6c4d616b5ae583b533e5e79","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","signature_id":"7f19b237d650c3e2c2dada758b0faf353ceddfaf46bd9c7e365fe5dd0616b720","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","signature_id":"87ca9ce38c81638d67c18762111fbc84a3f902586128fffc4d6a63e5d5d586d7","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","signature_id":"c301bcb0ad350b3e4edb18ab6b3112a4e915bcc1ccff4534d5bef4c12af07f76","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6676/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261481813905408","occurred_at":"2026-08-25T14:43:04Z","signature_id":"8570bc7030e47ad260a5bdf4a8d1e6ec729b76caed85d2d85acd384bcef9aaf5","pr_number":6676,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6676/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261481813905408","occurred_at":"2026-08-25T14:43:04Z","signature_id":"adc58fbe078ffd0b9a7c1502fc926ccd76f020ceef6eb743ef3c165775530e5a","pr_number":6676,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"2326e62e81a715665c83cf8208b5b992003fcc3c884e154ce68d5a2525a2a304","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"a0a72a832973622295dca8311edc8acdd0d1be0a93774aa5a3c2c41f0a225677","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"b5155d1abd3afd29cd7d2bb03f43d6d1523d798062871307fb209c787f593dc6","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"bd2bd12402128b3e6c7b2c7dc8891fd4bb2cbbd5fb1f6cc30af3b1c2b3e2de09","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"ca98a15fd3dbeb3c9c2ada441d28414afe433e197fe77478335816c4c14eb15a","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260279839625216","occurred_at":"2026-08-25T14:38:17Z","signature_id":"40844011a6d47c6fdea97725589c77bfe98a0d94d59281c86c002f1a0792c273","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6587/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253504889425920","occurred_at":"2026-08-25T14:11:22Z","signature_id":"4d865de5fe9dfc533ed0216b192b218fb0191025036e4ded553b013e74cb84a3","pr_number":6587,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","signature_id":"0dd4ca7c298b5c4afeaab0da7cacaeb92e7380f6b13a23bb92ff6fc133819fd1","pr_number":6630,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092249468303839232","occurred_at":"2026-08-25T13:55:13Z","signature_id":"02ce83b411d6d3eadebb040a79dc7e0fc19f5dc8037f2c40f8fbbf51c121a5e0","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092249468303839232","occurred_at":"2026-08-25T13:55:13Z","signature_id":"59eb8e67410e16bbb2ec63be568c198772ad94ee9f1c1cbda4a99f3c7987422a","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092249468303839232","occurred_at":"2026-08-25T13:55:13Z","signature_id":"7bd5b0bf2a08febff61bb7112c5fa60448aafb6f03c45e0ecefda98be25fcc90","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092249468303839232","occurred_at":"2026-08-25T13:55:13Z","signature_id":"c1c5c1541665e5683a94093cd7f6588bce5f0027f1e226c12d34e5a06ef243a7","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6677/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092248818148970496","occurred_at":"2026-08-25T13:52:37Z","signature_id":"0fc6eaf247bde243614d205434c32f36a568acb9c7fa2017a78c81ce9292cbf1","pr_number":6677,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6677/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092248818148970496","occurred_at":"2026-08-25T13:52:37Z","signature_id":"ebdc0b91a773f55107daffc829136d740ef7567c97d135876b3ab2f729e95efe","pr_number":6677,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"139f5384ed9c8675144c62f1bbdf582bf6ba3de5a1a2a7e69cafbaf9e1324971","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"3bedaad2d7a72d761e92c1a0c1322773388504505b8f7b830d9c172d097aab52","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"4a2ed9f21c01469e954cb46911f58a6903c7bef2bb37ab47e1b4136d17c074f9","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"52e3a2d01cede408cbe7165d526befd88f9e9f4dbcdc8aaa12bb61402619c205","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"8bd3d86b83f9075e0c109be0d76439a6735162556e0e3886ba813816b230c868","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"abc8216563c683da1f6e16ae575f37dd3f305f261e135e096e5da20727d96771","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"f096e2894e5607754bc52d3997abbbaf63ee7b84458d5fadbc6b046423af77c3","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092231268828712960","occurred_at":"2026-08-25T12:43:00Z","signature_id":"88ef5278beac4e0b8897706c7e61f4cbb2e358d71c83d41a4ebf5e0e7c99a5ec","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092231268828712960","occurred_at":"2026-08-25T12:43:00Z","signature_id":"c3e9e8fbb2c80bbe0532378e51b9ba35394e8a18321158765e0e9d2c9531e792","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092231268828712960","occurred_at":"2026-08-25T12:43:00Z","signature_id":"cba54857857016b020e86b6a86de63b4978237a179d12aeabfff1cca95a87fa0","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197879035727872","occurred_at":"2026-08-25T10:30:20Z","signature_id":"ba4575d0d05801721860d214c9ab20325f6c0ce684b7a86f371bbdb0b3c221f6","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"30b876ac66882fc0e335572b0826b8ec58b54d7e36dc27655838b99b58cd8eb0","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"58a3af70c95169637c5e6f6aa6a771a555147df7756a6c67c6c7a78311b59eee","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","signature_id":"59336122fa10346af3e78ba303a4d90407e92e02b272f6b3005e01628acb96b4","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","signature_id":"79c7c41c97eedc396d88101d613a41fb1d71de25afa00931594fcc8ea799c229","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","signature_id":"938d48f5f5615798cf28a4204904814739ed9c8d4708bcdf50c0a0e5b416c33a","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"16c6bbce9d00cd3fdb259dff38344220eb8bc09eaf6c7024a6b9039971482f59","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146681486249984","occurred_at":"2026-08-25T07:06:53Z","signature_id":"e77ed3272a3aa5a99a10d9c5ad6121981ac3eb73b228a8dd38fba0c6882dbb91","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092142908357480448","occurred_at":"2026-08-25T06:51:54Z","signature_id":"180ef015a082a3e691d738ac631464aaaf83f5b0ebf48a41b3ee60048cb89ac9","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","signature_id":"2c553608c906b4a9046fc287e40a200fb56683ae155754190ceb3113e9c01826","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","signature_id":"926bbbc4110caa69d86b22306ab7a9ad88bd051e3a6201884a30ef9e28c59b9b","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","signature_id":"df56cc165c01e6c8c810f48db72eecd1abb1ce36621379d823c8e5cb3996a5ed","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","signature_id":"f9e5e4b9fd79fc6bdafcb10977e537bfeabb8f238b48fc177d8dc722a58ba115","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092130874219827200","occurred_at":"2026-08-25T06:04:04Z","signature_id":"8d4d063103cc70a30037cefb230c7659ae5f161bfbe5aceb58476f2a4134c515","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092130874219827200","occurred_at":"2026-08-25T06:04:04Z","signature_id":"9ccbbc0b72f96cf97c329b914d4f5cc67ebf262aaacfa9b2db48024f1cbc3920","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092130874219827200","occurred_at":"2026-08-25T06:04:04Z","signature_id":"a71e16e9452ebbcb96b7843af9a8f55da51d63b8966fc237dc151347da86709b","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","signature_id":"82fe3afdd0fba60d3c7679d4a315c8480ecd5cf974435df02861bd04baa9a825","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","signature_id":"ac5746c88f40ad3ccf73d0e708c6422fe20ba547abaaaef12f47d1492ae2b838","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"7f995c509ce4acadbbdef98d81915a02d692abceffa2a81b563b98b98cf85466","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092044544542511104","occurred_at":"2026-08-25T00:21:02Z","signature_id":"83e3baeadd89b998c4b81d1bc0c70d07aa9fec97f2694533c8ed2432ea266d46","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092044544542511104","occurred_at":"2026-08-25T00:21:02Z","signature_id":"983555e8ae88df4ba006bdd3a98018885a3dc5949799b00108d57edd8321b6fb","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092044544542511104","occurred_at":"2026-08-25T00:21:02Z","signature_id":"f1a6c7858d28eeb824a65797f9f24cbd2dea977f3e661a9e3c7f104a3dc601ee","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092034671041843200","occurred_at":"2026-08-24T23:41:48Z","signature_id":"54a530c9876fc008fec6ac6899f443f3e3f5628a64422efb1e6064e8186af6e1","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092020242174709760","occurred_at":"2026-08-24T22:44:28Z","signature_id":"e6a9db1c5a3cafd8f545d8476a1e6b49dd9166a98aa20d59709936d40b1c52c0","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6663/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092019318316339200","occurred_at":"2026-08-24T22:40:47Z","signature_id":"a6a8523a32b9fba91a90fbd04abe90465e11b86cdf06949b929a37a85e4f40ad","pr_number":6663,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6663/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092019318316339200","occurred_at":"2026-08-24T22:40:47Z","signature_id":"e6285c4c12e9e0c85b5ecc25058de40b9f05b37d1a08c8a8c4a66108766b8ee0","pr_number":6663,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092016236664721408","occurred_at":"2026-08-24T22:28:33Z","signature_id":"4549c28d9f346a4101b2d199c56363a0a36787b6952f32440182d5210d9903d4","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092016236664721408","occurred_at":"2026-08-24T22:28:33Z","signature_id":"916b71cbe1c0282bebb7706c18cf185456f94291c6c7059f8d673bee6920e761","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"49911eb59aa73a3d7ebf02bdb4e56de7a616897c72e3ac7abe420c0c7a47e40e","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"6487190a1499ee6eb043aa40a245add60c110f31922a104a2b7ccae5c75948f8","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"7cc1252e9b8fe966bbe77836b76cbb84f92a735b546bfadc6681cd7399872b22","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"a9397a1f0aaba5d8ad982fe383aca1267736bcbd0d40f55741f427e85ee8b6d8","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"c5d8b0e5c56e2c31565dc9e53627bd3252420239c844111c2d9db849f9e9c51c","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"deae4888a32973190e55fe6f1f2ac8a8941dd6efae21aa17eaa50cc4448f5b70","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"7451289a87442818c6daa03144d0fb9efebafa88d3687d31f99e8439ca741e97","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091965378614792192","occurred_at":"2026-08-24T19:06:19Z","signature_id":"7830c998ab91a0251740ef30429a36f78ee908a1781d172a1f172ebef622c56d","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963250454302720","occurred_at":"2026-08-24T18:58:00Z","signature_id":"75d7cd315bc777c1079b9374c7a2b4aabd61d42a04817f2523f7ea7dab1326a9","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"2021d27012220f14a5eaeb1e574cf0e0cf00596ef715d56d7ab033d1204d0ed5","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"dea6785389a59c2a975d6ddfe69a85996e881ce704015e7a9c41548aeaca0017","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","signature_id":"19ba2327e177a466c74ef48f9d128288b644237f0c8199be7309ae0bf40f5a82","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932257060130816","occurred_at":"2026-08-24T16:54:39Z","signature_id":"70c12268048fb1a990e1fcfe48d8d53530e6791148ebcbe91068d54355e50247","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"688dcfad091ac3bb942f27fcd5e2e3d6471f49c9042338c1c965ab6c339ae7e6","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"d1005ca6dc326e3c744c86ae1e9fb1086de2e12e4a6351183145527b31e5eccf","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"db07fb00cb7c1dc1d49041e7a151eab39b4fee2bc9901dd00a1fc182625826d2","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"40b147b3a09185f7fbbd5416adf8d0e158c67796d5387c2aa465c0a43fd1dc30","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"8d53ed6b1258502dfa6cecd4e6049f72772418b7686ae5c9e29c6adc65e73c79","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","signature_id":"fd285a3626d258dcb1ba2d99b9eb4762dc5e5af32183bf400bd799c872dd2fda","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"94f9ba782dfa48d054a8eb8de04c3bde301c62297127cae4d03aeb5f65b1b3b4","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"e6714abbbcfe1d54c45ec38583b6367bc1667d055485de6ce77544f68acc9679","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","signature_id":"ccd1c15e8b2737a8cbb30d07ada8f1b6f0748afd9d43b748ec4128f7a61ce044","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879954055696384","occurred_at":"2026-08-24T13:27:00Z","signature_id":"b66537b24ffd588d399e5e7cc810bf3a188e5e0dce669b0b16956a016a18a10c","pr_number":6429,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879927736438784","occurred_at":"2026-08-24T13:26:52Z","signature_id":"6a060f99c27004cf01810556d4b6e1126f6aeee13d0cc7b3c046327b6e9bcc0d","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","signature_id":"25baffda5fc6f88c385cae06e401a6f4815895c798a9d1e087a18364db777535","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6555/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091237855425728512","occurred_at":"2026-08-22T18:55:31Z","signature_id":"df6c1e4a9641386a7eb5a26baa92084f62c849def92ac231cd5d430c67b19146","pr_number":6555,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091230843602210816","occurred_at":"2026-08-22T18:27:40Z","signature_id":"0420a23f54b484bd5c42b458707ed3ab40693ba805498945d41e0e64e6ee0507","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091230843602210816","occurred_at":"2026-08-22T18:27:40Z","signature_id":"1f603ba8c82583eec0f2843da23cf36993226ade45076c14bee8cc9c3b4c788e","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091230843602210816","occurred_at":"2026-08-22T18:27:40Z","signature_id":"9534a025ea1fda5ac6df21102fd48d9c182b38eaf8806344cd04c3af0afa585d","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090577382321688576","occurred_at":"2026-08-20T23:10:58Z","signature_id":"5715aefa195ea2179c48b8cdb3c0646c0b4dd4ee7db7b4e95085b994ec083207","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090554196767019008","occurred_at":"2026-08-20T21:38:55Z","signature_id":"3841200b7547f214afafd0c0c55361fa1f72cacddfaad20a9fcd5f4fb1711cd9","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6555/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090233576732758016","occurred_at":"2026-08-20T00:24:53Z","signature_id":"b4753d3fccec6f82fdb286c98c9fe8175786c7811cebd07b1f3eec79c958bbe0","pr_number":6555,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090166116277030912","occurred_at":"2026-08-19T19:56:50Z","signature_id":"d60a5810e046e63b38869d3666f37ae08c3d445efbce501ba3a5a908dfb76790","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090093258611888128","occurred_at":"2026-08-19T15:07:19Z","signature_id":"dbcf091eba0b35144538bee3334ec2a7d62e240fcf6f751b27ae3686de0d02e8","pr_number":6584,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4b645ca11afad5b79ec9dae25eb364818cdce5c3fe97c8a6de8581921ac37e5b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":105,"runs_affected":21,"failed_runs":21,"run_impact_percent":3.312302839116719,"anchor_occurrences":105,"after_last_push_count":90,"also_in":["INT","PROD","STG"],"trend_counts":[26,14,33,40,14,0,1,3],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":86,"prior_last_seen_at":"2026-08-22T08:48:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":7},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a default cluster and verify KAS is accessible via both public shared ingress and private Swift networking from worker nodes","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS, private ingress, and private KeyVault and verify all are operational","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP creates on-demand backup after KMS key rotation completes","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ff2780\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6hgvwqwzvj5t\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2eba795a-0e4a-4269-8001-ab35fdb2081e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2eba795a-0e4a-4269-8001-ab35fdb2081e\",\n      \"name\": \"2eba795a-0e4a-4269-8001-ab35fdb2081e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:39.227173208Z\",\n      \"endTime\": \"2026-08-25T16:40:48.587074479Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ff2780\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6hgvwqwzvj5t\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2eba795a-0e4a-4269-8001-ab35fdb2081e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2eba795a-0e4a-4269-8001-ab35fdb2081e\",\n      \"name\": \"2eba795a-0e4a-4269-8001-ab35fdb2081e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:39.227173208Z\",\n      \"endTime\": \"2026-08-25T16:40:48.587074479Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001002420\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-t2kb5kkgwfmt\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\",\n      \"name\": \"eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:26.515094904Z\",\n      \"endTime\": \"2026-08-25T16:40:28.648310137Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001002420\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-t2kb5kkgwfmt\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\",\n      \"name\": \"eef968d7-3821-4aeb-a3bc-7e141d9d4b7b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:26.515094904Z\",\n      \"endTime\": \"2026-08-25T16:40:28.648310137Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--kqf9svj6lsq5/cluster-hs-pre-xwpsbd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e8a8a0\u003e: \n    failed to create HCP cluster cluster-hs-pre-xwpsbd: failed waiting for cluster=\"cluster-hs-pre-xwpsbd\" in resourcegroup=\"rg-hs-pre--kqf9svj6lsq5\" to finish creating: GET https://rp.j6292864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0314e9a-f076-4160-9b33-87341813da0e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0314e9a-f076-4160-9b33-87341813da0e\",\n      \"name\": \"f0314e9a-f076-4160-9b33-87341813da0e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T07:56:21.373383208Z\",\n      \"endTime\": \"2026-08-25T08:15:30.901546963Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--kqf9svj6lsq5/cluster-hs-pre-xwpsbd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e8a8a0\u003e: \n    failed to create HCP cluster cluster-hs-pre-xwpsbd: failed waiting for cluster=\"cluster-hs-pre-xwpsbd\" in resourcegroup=\"rg-hs-pre--kqf9svj6lsq5\" to finish creating: GET https://rp.j6292864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0314e9a-f076-4160-9b33-87341813da0e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0314e9a-f076-4160-9b33-87341813da0e\",\n      \"name\": \"f0314e9a-f076-4160-9b33-87341813da0e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T07:56:21.373383208Z\",\n      \"endTime\": \"2026-08-25T08:15:30.901546963Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"460e7645261b38bfd4a8103a388c161c308276756d9b8ea62787b9b5009d2135","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"4e02273c25987feb6c9bf4b5b1d1d5fd920f82e15dc281a89db33f4a9aed76a9","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146717536292864","occurred_at":"2026-08-25T07:07:02Z","signature_id":"6d98542146857385e0ad2f780a80a61651f234191e5005f754beacbb7ea51a34","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"9806f10f5f28de5b459cf7d30f0fa456f7ab916323bce3ad88103d4bb5b0c102","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091085161763442688","occurred_at":"2026-08-22T08:48:46Z","signature_id":"96ba3bbe47b7611d618631961d3c5569fbac980522d504f0be58ba9cb697daf6","pr_number":6640,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"0624e47ebc01259a86393f6b94bf7768700270439ec6340fe1971fa2e93fe976","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"1452f7adbf81aa74b093ca45c0fa8ac97627c68d98fe9e5920a5f92b6a914584","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"2e4c8f866988abe4f3fbe1b08e877e0cfcec027a2f7cac659c081bcd21149734","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"59c1e1347bd427d8d4783d93b76ae2027a93d4ee3213b2ac712d69c81a638899","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"5d6538d0941a77db5c74d2366ce1aa2770649359b913f22f6200ba4597a39ac4","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"63db49341df819db34eca5b5ac261b93de3b59ce36b9b3d9cc9b869199075023","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"736aa2a9bbb3e9cffb2dbbf9d439c8108f35bc75e1a24329e387a56a5967bf82","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"b07ba0edef0c9c36e19410fbdad5237fb04ec9614b2c3154f0326e6c02187e76","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"dba72741ac72f22b446e90bc10eae8af85de2d4bc3c2708c1e311e42dca76710","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"e20c717ced12823cf99b365d56e4c2a886f13b4deb5bcd5d3e3e323a6edc8827","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"f60c643736c76a179834c7fa2cab4c02555094c1b8f19768ccb5a570ec769e60","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090960143322714112","occurred_at":"2026-08-22T00:32:00Z","signature_id":"1acede20c12184d185bc2dc16bc7034c0bbd254bcd41cbb5575e6ffd0a010dca","pr_number":6639,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090958470403592192","occurred_at":"2026-08-22T00:25:22Z","signature_id":"b4f721e5ea0f91bfbe7985abc52715ae48810e5fe0ee1d0cb9f7f8d919e916b2","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"20d6beb16979adfe257392f9563b7f0ce21b67ae5819ce4215adfcbc2687c77e","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"24bce759fd4066d1fc588db89a49afde37cbc6f44bb5f6aba66d9a685d775c4d","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"38b585818bacd259d2b2d3469bb56025bdb56c75d07786458c1a90fc46168739","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"4400d5437d5e17e6ebaf9c42924e7b9494fbb3607ad67e3b76de4540150f3be5","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"558ce75781a063d32559631db7e9f63baa11cbaa0afcc8f29782f0c9b26502d8","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"612bc332a79ff67dcd08cf1d8fb4be9d1085474e2e3993b2e0eefe78d68f6752","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"7a0a1e4f4e85d34d06ec7c677ae0c840a32378750971186a367e902850bc07de","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"80fe67f4f734e23425e71a187d1fa480f02fab99935a1c2267e4f7e705acd284","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"8e6bf6dd5a581206dcb87e48ecf50a24fa31dd1fd66ba0ba42db56ffebb2cd6c","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"a6df34000b844d5b45630664d5b48bba8d389682ea434496b73521132685d440","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"a9c19efdb9bd31e434c2b860e318675d4fe13ce36666fea13eb0e62e55f757a3","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"ad8dfe8ab83d51da65319f6aaa59b17ad6fd532c019b7bd9c946b6f18083edc6","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"c4bfe25397b8b2d437eae4569a7723b855e9ee12034118a652c42c8be6cdba5c","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"e26ff3c2c390f7d5fcc4d0231fd0d98319d7438d1526b577703513ffe3587558","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"e624b5aa4b7740b1ba645384a0a92d298c7cf268038db0a9142479ea68b268db","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"5314a86bba0bb9f12a61be0bd6791fecc87d5f8d00fa3914bada32d665291e4a","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"5abca12243bae993f07f4e4e63a4742fb8d2e7c8e46fcd64385eec5fdb894b7c","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"6210f5fef10139baa9478a3153593fb949e25d7abffb54ffa7e12322cda4088b","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"6f0420fd84447d47b60aa6ff6601d0849f8dbf938905c55ca4009f1b4d455a72","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"98685d5e8b443df38e924eb75184bfca6ba5d2dd3e69379f57c60863b5eaae87","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"9db5b43574c9db1eac22e7d04f83f6d559414b29d58cf3741819a9aba0e41e9d","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"a57b62065dbf4f89ca41062220a43ae387245fca845dcefa7c5564685c21c452","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"b47de51b0f358050d51a602d396e7136be355a35482bfffa6484fba844fcbc14","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"c000eccb673f0180f2f3508c516881b27fe7b2013749e38d8cd16f2c313d4a06","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"f086b4934e2f69cee46a0807f947e11514153d16a601e33d6288b4153bf2445b","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"fd35d176d4a2777f7fd1120d1d75ce1502e674361791c7c82a8b9d0642a8b75c","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"014b0124a0340b2d0058f92137c1af01c74b55c601a11f17b66726689e079b7e","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"060e9895c1ea9b171d554719ff54ac5c009daa12670fadcf5136df441b33912d","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"65a5ada1bbaf672403f3ea6bfd30e59064f627383b60bd9b883c7505c21cfa14","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"6c0b36b89e89d4e3268ffb83a5695f5ec11294751941582771219733b9d66cfc","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"7ce3cbe97b9c16950bd30a5c6ecdc0d15277684addd5eaeaab7d9097b895113c","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"b3e2ca6406b7d0c6d46b39472954b647cf1e029d807df155f8090c7730519eb8","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"c257cfe9f4f89612bb73d7dcdf0bea39effe21ea7a9e7e339d581803c506f2a3","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"d1938d6322253f9c816c3bf43e93a7b6a173eeb9c6980a4b792ae7965b363f77","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"de3b527ad7839cd28018fb7bc4ecfad91f38d4772c6b2e873fde7ed665f9392e","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"e0211790082612284639b917f2152b2d4179b71035c60ed504f9fa103648d359","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"f9e689ec1684b8942c33d33f9c825e3407a1998990b5718a09ae5147fa362099","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"09094648223231bb9dbde764787354bcdc986a4c2d7c90b0773b64e1f0744693","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"430ca5b576d22528c6770eacf2a6c8f97dab34d25d271c47fd31c6ae0cdf9b69","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"b709b68e31b4d438b7979bfe9e931a7b3e4f89ed2b8708048c02a5b68f58e960","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"04bcd6a06d29057b10be12e04816d64f01f1530b8ecf366646c79fb09d1cdf11","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"246c803b1e707a889593bb8ea446899c00d7d4d657aab923a9e9edfe4145c115","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"5bea59fcaf64de56a61dc5fd611024a94b77bcc55b77b15df0fda9ff595af14d","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"5d468fd6480dcdcd43dc7d63806754d69878da96ff0fa95f52d664649c52df57","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"8c4a7fbbe76c05848c2f39efbe44879deba3e853f5ad0376ee17a8c4078e8717","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"9c130e71e529e7927b1ef5616dfea401f80d02d3d228f55ea0a3620ca0df430b","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"c2a1bc6923afa666a0125d5314a52ffcd5da96f4286af2c534bd486ed9e26327","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"da3035a89dd9906e064acdf1fe56dc3f4a9a46899ce1afec10fcf46eeb40e518","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"dcb25a94761633dbb22f75fbad3c9984bd81a1f744d278223f859c52de991ad2","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"dffa1f46e9b78e5e92a4bf03da03eebd9cae7182e52521b5d9053ab0ee31762c","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"ed778a116fa87cc48eef03343ad5455868c45c5e4eaf677d04ba06620101bcbe","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"20ef09c3c97235c2e9f93c63c6114435707b1b4b0ac57f2871c9b9d4a0196473","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"43aac46da268d3089c276a3a7631b3948dfbfd971513e242bb5454c0ec6d3645","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"539ccbc9dbd92a40a1093e109cc7a09e024aedf2ea6c2db8584640c0a28c1013","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"5b42f898212fc24fb029dec13a60a0660c04dd62977d06a96fb0ecfe94159be2","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"601387522ed142b50871a3c00bc33122ab7c4cb9f2d93fc05df3d55cf90311af","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"7ee8f0811965f3a76e4945f432b1a1c826430f4d21048c68406255e9480f033c","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"be2cef2f811fde74db3929e22ea469291a511bb9389c23ca64d680bbdd1109ec","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"ce6a1cf5235a0c582fe2a45b9ca1098300fd89148eeebdd597312a281ed9b8da","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"d09b7d40ce5ea9e928c47d69506c3e959ef1199d4abc5ccde3d3ee558fbb4613","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090497381098328064","occurred_at":"2026-08-20T17:53:09Z","signature_id":"8eea666df51a57e86a1aa975db5a69225adbc526d8483f639ecc78381a317867","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090465920358027264","occurred_at":"2026-08-20T15:48:08Z","signature_id":"fd4d79833e2c9a4f42e2fba91ee43b4d71c5bda4c82416b82d7173763d24d9cf","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090405439429677056","occurred_at":"2026-08-20T11:47:38Z","signature_id":"c32c7397b025363d964e7dd9b9c594012cac3c74f81e392ab4a145c1c243c792","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"00cef274ddffb2732acc632cc1b182ec59fb565f02720c8b67051535ce772f83","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"2f55155f7b22fa74558cdf44004f4500ffdc59a95b789699899d148245d4d673","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"748b6ba9d3c5cd5a2f3579256ea3a7348e7ff1d2c5d3c8e2870e22efa3a6878e","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"8a9137894c721007ad4bf245bb688d32d3743ef19d01d7b13ceab27d2d7bd6db","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"a94744931a24c357dec9cf5c8170c369eabfba697bd1a7a95d81dc746805f80b","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"ac7c80c51e0db130adc148de4ff3c42257383d067d073129335c97f399ab326f","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"b54898cff87ba825361731cad5c3d16a6aa7cf7febfbc0f3c0924bf4bf06be42","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"b97950402c2638225fdede0ae11cc1339025f1866c657ebfbd50ae5666f586f0","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"d000285be635d8d22977be53c3a079b5b8b94ed11e8d9d6b46e9e0ec678b4a9c","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"d8c2c2dea9d1fccad931eccb2040ed9768b2ec564e3e703465698e8ec24f3692","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090191599463043072","occurred_at":"2026-08-19T21:38:05Z","signature_id":"47752b32579446bf4bfc4d096c1e5033967d1b7f583b8cbd0d426ad63ef21752","pr_number":6608,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090133175878553600","occurred_at":"2026-08-19T17:45:52Z","signature_id":"6f47ddd82e439f89e71758068a026c71acf578ab4e0cfa1e4da4103013b9a768","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085264507539456","occurred_at":"2026-08-19T14:35:33Z","signature_id":"d688cab4cfde23219f6001dad334c01cc37a31b4faf222dcc80630c4ba631dcb","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085264507539456","occurred_at":"2026-08-19T14:35:33Z","signature_id":"f316e727f1813330d33076f4a020c08fdc5b05e75a46acf94630a96035d90fcc","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"08288e83cb5aad9a4943f319e2ddda651d5ffe0ace3f2d0d802c3d560cdd2ef9","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"2a405dc6a6ba24177e78ccdf06b8833de0334419e3f0404f8301d9994587b574","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"516a9fb731b4f124da567dae4c2c2ec5a10033fc9b93c35c3ed7afe5b88c39fb","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"559acce7c1322e003687e061addb4f08d87fa4fe5fb407f3748561a319b962b1","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"5edc6d0663ee0be14c9cb5d0ec6ac6990a7c5b0ab006bbf59dec57df7c41383d","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"8285d0d5b2c0708cff4391fb99a39ffc99f1f8527c748614b04ae6c318ae8193","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"9f068f29f402d90a1d62ce54b8fd412bf10df785fb8156d8efe3ed8c5d9ebd81","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"e4969ed09fecf108227e0a76e171ac4453c2b4a62d9b74ef9ddb916c6b9dc192","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"ebe92ac8ac5fe50b70fcb288bbf69fbcdf672537e9a80d11d7a2de1dfbd2bac9","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"edbee3da9aceb6c190f8576e1b5366014cb172c4899e5874bb8d9f28ed57eec9","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":89,"runs_affected":89,"failed_runs":89,"run_impact_percent":14.037854889589905,"anchor_occurrences":89,"after_last_push_count":72,"trend_counts":[14,9,14,12,2,0,14,38],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":386,"prior_last_seen_at":"2026-08-22T05:45:38Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":89}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T20:01:01Z\n  Ended:       2026-08-25T20:16:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0886784-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdq6r93avnmkufpa2dlhkdn5bhh4958-c6q7t8b4b8e6m3v\", pod=\"router-7dffdfb86b-r9dqt\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sdq6r93avnmkufpa2dlhkdn5bhh4958-c6q7t8b4b8e6m3v/router-7dffdfb86b-r9dqt has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-25T20:01:01Z\n  Ended:       2026-08-25T20:17:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0886784-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdq785qenoju9gska8l0aimh5kanj73-r9g0o6t5l6s9a3i\", pod=\"router-87ddfb859-rc8wd\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sdq785qenoju9gska8l0aimh5kanj73-r9g0o6t5l6s9a3i/router-87ddfb859-rc8wd has been in a non-ready state for longer than 5 minutes.","alert fired 44 time(s) (4 unknown, 40 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T19:36:34Z\n  Ended:       2026-08-25T19:42:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8405376-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdppr507knoluq4b6simih0av16epoc-p8e6x0p1e8y3d3q\", pod=\"cluster-network-operator-7fc64bc864-z8m58\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sdppr507knoluq4b6simih0av16epoc-p8e6x0p1e8y3d3q/cluster-network-operator-7fc64bc864-z8m58 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-25T19:37:33Z\n  Ended:       2026-08-25T19:43:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8405376-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdppu4r7j3gvpcpphqjg7kspjjb2d5a-ea-listmdvtnb\", pod=\"cluster-network-operator-bc9698665-b8c6g\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sdppu4r7j3gvpcpphqjg7kspjjb2d5a-ea-listmdvtnb/cluster-network-operator-bc9698665-b8c6g has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-25T19:37:34Z\n  Ended:       2026-08-25T19:43:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8405376-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdppt1p2lo3ijsep2g5q057lthm1bnq-y8l2t6b7a3x5m5p\", pod=\"ingress-operator-9dfd956c9-slrm6\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sdppt1p2lo3ijsep2g5q057lthm1bnq-y8l2t6b7a3x5m5p/ingress-operator-9dfd956c9-slrm6 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-08-25T19:37:34Z\n  Ended:       2026-08-25T19:43:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8405376-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdppu4r7j3gvpcpphqjg7kspjjb2d5a-ea-listmdvtnb\", pod=\"ingress-operator-6f9986dfc-xd7sz\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sdppu4r7j3gvpcpphqjg7kspjjb2d5a-ea-listmdvtnb/ingress-operator-6f9986dfc-xd7sz has been in a non-ready state for longer than 5 minutes.","alert fired 15 time(s) (3 unknown, 12 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T19:38:10Z\n  Ended:       2026-08-25T19:44:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6756224-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdprcahq51n0d0h5krenqtt6q04qaja-t5m9k0x6n2j1y0j\", pod=\"cluster-image-registry-operator-6599c97d88-9k277\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sdprcahq51n0d0h5krenqtt6q04qaja-t5m9k0x6n2j1y0j/cluster-image-registry-operator-6599c97d88-9k277 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-25T19:38:10Z\n  Ended:       2026-08-25T20:11:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6756224-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdpr4u0kflvoq3tr6uonatusgjkqtgb-i7z1k6g5a6y6a9r\", pod=\"router-565466c4ff-x85nf\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sdpr4u0kflvoq3tr6uonatusgjkqtgb-i7z1k6g5a6y6a9r/router-565466c4ff-x85nf has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-25T19:38:10Z\n  Ended:       2026-08-25T19:44:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6756224-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sdpra59hpdand8n0tpnpdkp1smvi0if-b7k4i1r5j7n1v6e\", pod=\"ingress-operator-588fd99d97-qflbj\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sdpra59hpdand8n0tpnpdkp1smvi0if-b7k4i1r5j7n1v6e/ingress-operator-588fd99d97-qflbj has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"c8e2dae791d9eee6c2dfe89ff5eb24b232035b0f6a8f9722f2f09c4da4c46270","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"00cf1cdbbe1c25edacd565cc3dbeb30b1fc906ca4e493b4125b891e5503ffe76","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"670def82337f4e93346e218bb912d2f5102dda3d4e354be6f7a3ae52af837b74","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"f1a9a05cfde18cca1700edae4de1bd0b0a9bac2292183bc359385aab761dec79","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092295016859832320","occurred_at":"2026-08-25T16:56:19Z","signature_id":"e5fda378ff57d5b1940c8266f347e7ee86ada79c73f8812c6767fe80cbdefd69","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284063539793920","occurred_at":"2026-08-25T16:12:48Z","signature_id":"2d9cf619745584716b1d36a78f973479ee6f6a9974e831e0e6595f6f7c09c34e","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284042668937216","occurred_at":"2026-08-25T16:12:43Z","signature_id":"84c9f5cec5ace865d6364245a39389a7b4680b6bfdf2958173dfcfd15754e303","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092284038717902848","occurred_at":"2026-08-25T16:12:42Z","signature_id":"7a5c0ec1b1bbd3d90f56d54a6d79e27df4b0c9b8be003edb69252609bc761f85","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"fa55ad2520ac87c347911e5a9a91dd9cd00f969a783efd3c191a3eabc9b22503","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"945dcc48e93f7e5479b3db04e970b41db897299b7ad75aca01031fd96f7fcf6e","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","signature_id":"d5f85864d3d15c27d9d690ef128585ac4484e8c8af2bd1de02f0ebc7ecb160b9","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6676/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261481813905408","occurred_at":"2026-08-25T14:43:04Z","signature_id":"6619f8482211ef7b9931e9e6e5ce9917b4a89ba14b1e55a1266a1da1dc746d2a","pr_number":6676,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"85470de5379b7a58f4475a5818e36e604131a03a4654ffc58a012b2817701ddb","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260279839625216","occurred_at":"2026-08-25T14:38:17Z","signature_id":"ba0d4cca986340026ad28b58a3d7254c20c5a9e8dbe174366e6c0feb810b455d","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092258099980144640","occurred_at":"2026-08-25T14:29:37Z","signature_id":"c9fd5e19ecb8a19404c13852abed9dafbdc292ec71487776299a01961be38d2a","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092254086249320448","occurred_at":"2026-08-25T14:13:40Z","signature_id":"6fc405c92e08fa1576ff900b57b0c002219d66eb9e455d1fd3d73886dc969d02","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6587/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253504889425920","occurred_at":"2026-08-25T14:11:22Z","signature_id":"2cc3c6061c407d3c74ad5325b6c2ccb6ae669b0fee5d48f4299ed1a3a6b6f5c4","pr_number":6587,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","signature_id":"0a3222b13ca8e18b09f294cc8bf6c58f2b714b4f68671591fc81d113df214527","pr_number":6630,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092249468303839232","occurred_at":"2026-08-25T13:55:13Z","signature_id":"7d256492ee5379d7f038bf539080c373cfbf081cb07ec0c35738f57cdb906cc2","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6677/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092248818148970496","occurred_at":"2026-08-25T13:52:37Z","signature_id":"85d3fb950aa7a81bc280ad8111f550c48065298ea6e7ee23a30155e0451f6699","pr_number":6677,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092244166900715520","occurred_at":"2026-08-25T13:34:16Z","signature_id":"46fe202d87700d2141f9c24fa084e86f0be02779da003c0aadb556639c6b00c1","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"142358f238029e754285919ac9ae3371cf3d7fcc3f75f1908d60371d3bae5134","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"1ae78d946fc05772983eafd321d0a8f04e93c242870b0eb8a31923e9082da049","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197879035727872","occurred_at":"2026-08-25T10:30:20Z","signature_id":"9f099708799bf3e2ac428bc43b19f24600de3b38fc32adfc9aa785b8a28ba4cc","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"9323f34e5dc7624ff70491391f71624b8fccbc2b4b0fdf3ebe01e36965be3373","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"1399061bb15ba2afe3a3d49ef0c1c03b496dabeb92275fbed52214cd772f172d","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"ab1ad628fd91e461795e6817e8295d83b1b66efac12206305c9fb1793f4943a9","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","signature_id":"9a2d5923b47e722ec14a03cd82e80a5847eb6af6c56086cf441bc75f54feb7ce","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"783b9a11bba8d6f9b48cde87f4a3968184ce14d623eaa09a0d5aae21b1c10281","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"6791742669eeec9806dabaab1695d3466c0bad5f4d644012bab5a19c0d345acd","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"1418018ee02cdd89dd6ac2c5c6e1378a62f42c5f58ff6ea19b95f13d3e928b4a","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"e906e7ae8a41e3b71c67202b1c3d92e471ada05f8b75a59665632b01eb944b26","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146681486249984","occurred_at":"2026-08-25T07:06:53Z","signature_id":"85a4cfecb0adb1b2747bda34b1b1285829950a4778f69e05021f9596834b1154","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","signature_id":"7151c69efd10f0212e88001c4fa93856449529055b2b769b7fc63617db078de2","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092130874219827200","occurred_at":"2026-08-25T06:04:04Z","signature_id":"9912e6cb8637cbc60f5d14947b0d4860ec095a6f064bb2b7e514dd1160ef1a11","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","signature_id":"939ee2aa813aeb34b5caee82729b4311d14f106923472e3fd6fed1c4ae71c667","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"04107c06cdc1c97e2051bd30a01f9feaff1579d56b011fda20b5f0cc7435b337","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092044544542511104","occurred_at":"2026-08-25T00:21:02Z","signature_id":"51d2fea77be7cecb840acdb29e21e26279922de433103aefa5ec401ed53ade1f","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092034671041843200","occurred_at":"2026-08-24T23:41:48Z","signature_id":"2f1d9b2b8857e75d809a23da90c2e32bd0a70345c25f6862fc7292ac5c796db3","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092020242174709760","occurred_at":"2026-08-24T22:44:28Z","signature_id":"f8ca631bf4ba3de6a823751843028c53bf8f63552f781736fc0f06124e42925e","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"762c894801e83a205ad61ad7c1a1171fcbbd7db65d82ac38613bccfd72e8ae0d","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"5a62ef9a7062e35c1e10c6c46839368f20045e53d18429e0c93ad1b8118f4ed4","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"05c28635da15b6abb7fa52304cc82cedf7070115a67070cb7058c7bc17ceed55","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091965378614792192","occurred_at":"2026-08-24T19:06:19Z","signature_id":"d9b2b2f48eba30a813d1493c2cf9d779d74129e8e68690c29d48df3b77d5680a","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"84936854ba2393da3d6d4a90cbe465eac2bfa99410eb5ae262e1869d118d0d44","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","signature_id":"d6263c32bd22604b284f6288625ad3fcd30c678787956b94fe5bf7e9a8e0f6cc","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"ddb17f85d08dfa0f0d60af8e146b060c3f080bb3cb873c1fd946241da0c96c74","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091920407983558656","occurred_at":"2026-08-24T16:06:36Z","signature_id":"c52899aff7bf73e0f12d284cdf7344825014b3fd387fb2f270a8871271bc7acb","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","signature_id":"32b177871d4669aa26bebdb303a13d4eeafa32be58fe0c29eecd2b7355b315ff","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"d1d5a7d0ff60a9161bf6b8e8b3cd897227641812139ecab6ded5ff874a074bf7","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"41f9fa028fb65313589b4f31abf40b7c37b7c57dd66b9ac548b1148f02514739","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"d55af07fa2830b9ad7b195c673e4e375ec2e39f3e870f6e6e4139ca2af8abfa0","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6593/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091039068275085312","occurred_at":"2026-08-22T05:45:38Z","signature_id":"95aa3aca64e7cc46c2a08e24bb4eb93743dac610ac55bf5e95aee1631ffe4b61","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"b7be27a9228028688d416758c9450df4007cbf195a10011266f7041548c3fec5","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"14ed32dc0216996b1a8878b8a835543278abe81456380f713b5041f8fc293420","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090908436404375552","occurred_at":"2026-08-21T21:06:33Z","signature_id":"ec64a710749d633cd2c44431c9388abb2b8806231c9e0acb1122140d519a61e1","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090882234021056512","occurred_at":"2026-08-21T19:22:25Z","signature_id":"88b630622168ed4766114eb7b40abc17cff2f5a5bfa775cd4914781d0114c005","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6593/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090879591731171328","occurred_at":"2026-08-21T19:11:50Z","signature_id":"28a74d91f38fce537ecd7876c2fbeb1def25cdc046d90f155aa117593c40183d","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6652/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090868362149105664","occurred_at":"2026-08-21T18:27:18Z","signature_id":"cb9e738da98477afa6434f301921aa71901c190c0fa2ae1f4fdd3dc32d9b1f5b","pr_number":6652,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6650/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090866482933141504","occurred_at":"2026-08-21T18:19:50Z","signature_id":"059468d2108c80b7a8e95125cdb23fe8c965fe7682f802b5eaf7c270a982de71","pr_number":6650,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090836156110344192","occurred_at":"2026-08-21T16:19:20Z","signature_id":"1ee6bcd10080625d968e07eac093a0e8c09f764a459c0d9ed1e90b69d0f7986b","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"a05ad038bf9aac70e32cc0368923a71fd08a6b2128f0cdc785f1671acd4c9134","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090753518062276608","occurred_at":"2026-08-21T10:50:57Z","signature_id":"d637ee2c2450b4d6a5abdfb0894110693323297ee0ace44237471e7cf0e1d59a","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"d1fa73718ab2519b4a0e1216040040fd07bcc0f880baf692f5762be6551c9a56","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"ed86faacee79b1d1d24720026b790d34d8418a0d8388cdee3111523a76e60e24","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"59e480ebe494d162afecad172ddf865743624a68f83750f3f473d533160b3b19","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090577382321688576","occurred_at":"2026-08-20T23:10:58Z","signature_id":"f3708a10e3f19aa3eaca1794571c4aaec444f1598c47dd25001400f3078b89a8","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090576104745078784","occurred_at":"2026-08-20T23:05:59Z","signature_id":"6fb8e1ab71971b31e568d57640f39efae16402260414e8b3aa699cc7b204bdf9","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"3468b8c2c816f1903b03c85d762cf81679135c586b96ea19bc866ef680ab082d","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090549448319963136","occurred_at":"2026-08-20T21:20:03Z","signature_id":"ffeecb88b31cc411dc8045e435c12c5f29ddf0da822c623c2e41b8ba06d17b7b","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514755771961344","occurred_at":"2026-08-20T19:02:12Z","signature_id":"06dd656267c6d34d26f117c02a8ccbe440b9f064addf99cfd5b363c42f885b3d","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"9d20cffd8d432102b61dbbd4e1ce50696352d2e43889ac63d6df80980c8d111d","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090489518846119936","occurred_at":"2026-08-20T17:21:54Z","signature_id":"c7ca06524fabe9e64053c271f8f0f6a6a5830b6427b478c52145ffa5f10b5b60","pr_number":6278,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090419688172425216","occurred_at":"2026-08-20T12:44:10Z","signature_id":"b7647a84afd55a9c67574cc6ec4034d30cef22f2363e4eff8c48a90300df4629","pr_number":6617,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090389808017838080","occurred_at":"2026-08-20T10:45:42Z","signature_id":"8c462b89c57a050fa84108ca0066a350520f68f99013cdd5ab3d20bf880b0222","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6616/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090329417594179584","occurred_at":"2026-08-20T06:45:44Z","signature_id":"24a02abde95219099d1524e2e5aa7008b06f1e9672a9fe6c323a562037eb7f62","pr_number":6616,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090313734160388096","occurred_at":"2026-08-20T05:43:24Z","signature_id":"f2489c96523fd094164119ff2a1d32086287ac39d481d98edb42a30691de61f6","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090249792243372032","occurred_at":"2026-08-20T01:29:20Z","signature_id":"e0ceb031e603349f821d9e164278112ab870c2f3a5b4a2121c737e4d31ff2768","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"42dbbb683d20ebb408cfab3ddf9a96496e42a39b8db6630d20713d16fa7fec0a","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6555/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090233576732758016","occurred_at":"2026-08-20T00:24:53Z","signature_id":"bf238d0ad10d36467eefe0d07977bf4d6a9dd12b68f3e414cb869ce3d72349cc","pr_number":6555,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090221977028530176","occurred_at":"2026-08-19T23:38:48Z","signature_id":"f1fdf1ff2fb718d1c8a9ff25e58ecafe17e2abf2d47632186ffc24c15fb3ebfd","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090191599463043072","occurred_at":"2026-08-19T21:38:05Z","signature_id":"01966e52482dd9a1375c8351a3593e5fad146e637a4c9387fd4a038652d5eeac","pr_number":6608,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090166116277030912","occurred_at":"2026-08-19T19:56:50Z","signature_id":"d8fd3c206c23d36dd54b4c1a854933e286dfbc3d5cfddcfad693ea3995863e32","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"d72e24ff9b0c8a737023df49a3efa6985c6c8132f311c961219684a83d87232a","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"172fe634a5279b7fca05fd5f9589d216e428a0b4959f4ad85d7e59d71aa0ee3c","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085264507539456","occurred_at":"2026-08-19T14:35:33Z","signature_id":"611225ad9ec7a09f32026a1d1a439e7cdeea3d31b702dbb4b07bf74fdf162705","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"a0a91a26492bf825ff807684676ca7b8921438ad4fa4db49f7fa7c17c3799856","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090076575402823680","occurred_at":"2026-08-19T14:00:40Z","signature_id":"03e27710d2fff2dd7d16273a4334ebe0c89597d01d5530e00cb0fd71e159e501","pr_number":6606,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6600/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090054287706034176","occurred_at":"2026-08-19T12:32:25Z","signature_id":"e5a51d0b1db4e0390900606b4a4496ba6e219f60f9b9127ac25cadd7f467125f","pr_number":6600,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":80,"runs_affected":80,"failed_runs":80,"run_impact_percent":12.618296529968454,"anchor_occurrences":80,"after_last_push_count":49,"trend_counts":[21,23,33,24,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":149,"prior_last_seen_at":"2026-08-21T18:27:18Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":80}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-21T19:35:28Z\n  Ended:       2026-08-21T19:44:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j9105664-svc\", component=\"frontend\", method=\"get\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/externalauths/{externalauthname}\", severity=\"warning\"\n  Description: The 99th percentile of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/externalauths/{externalauthname} has exceeded 1 second over the past 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-21T19:36:27Z\n  Ended:       2026-08-21T19:45:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j9105664-svc\", component=\"frontend\", method=\"delete\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The 99th percentile of frontend request latency for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 1 second over the past 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-21T19:36:27Z\n  Ended:       2026-08-21T19:43:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j9105664-svc\", component=\"frontend\", method=\"get\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n  Description: The 99th percentile of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} has exceeded 1 second over the past 5 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-21T19:38:31Z\n  Ended:       2026-08-21T19:46:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j3141504-svc\", component=\"frontend\", method=\"get\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The 99th percentile of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 1 second over the past 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-21T19:39:30Z\n  Ended:       2026-08-21T19:47:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j3141504-svc\", component=\"frontend\", method=\"get\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n  Description: The 99th percentile of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters has exceeded 1 second over the past 5 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-21T17:10:50Z\n  Ended:       2026-08-21T17:18:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j1918464-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The 99th percentile of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 1 second over the past 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-21T17:27:51Z\n  Ended:       2026-08-21T17:33:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j1918464-svc\", component=\"frontend\", method=\"post\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential\", severity=\"warning\"\n  Description: The 99th percentile of frontend request latency for post /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential has exceeded 1 second over the past 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6652/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090868362149105664","occurred_at":"2026-08-21T18:27:18Z","signature_id":"9ccad5c6abc73817f266002aafa0e53cefc97f74979f093344241981950c9562","pr_number":6652,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6650/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090866482933141504","occurred_at":"2026-08-21T18:19:50Z","signature_id":"80a92b3760718096cd754f61c5e35a5af2d9b3135a3e9fe48e23b48a27c4d6f7","pr_number":6650,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090837156841918464","occurred_at":"2026-08-21T16:23:18Z","signature_id":"dc9a448fa5fa6e2fcf21e4a673de64ca41bd972130cd2d303b365ed97abaad78","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090833818511478784","occurred_at":"2026-08-21T16:10:02Z","signature_id":"dc9a448fa5fa6e2fcf21e4a673de64ca41bd972130cd2d303b365ed97abaad78","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090833794218070016","occurred_at":"2026-08-21T16:09:56Z","signature_id":"fcec3585b17892e01a8a8d3acce94709e69a2c001f4b79a074973e51b2eafdf9","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090803404556210176","occurred_at":"2026-08-21T14:09:11Z","signature_id":"e0137a945f1e2a54ce81a232b4041838395e6a1684d4fcae4f95e8d894658012","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090803025537929216","occurred_at":"2026-08-21T14:07:41Z","signature_id":"497befd8d2afd386b85a479e07ff0f9765dd27d9ab898de2a689c72a9da46088","pr_number":6646,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6647/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090794933425278976","occurred_at":"2026-08-21T13:35:19Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":6647,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090753518062276608","occurred_at":"2026-08-21T10:50:57Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090745162878685184","occurred_at":"2026-08-21T10:17:45Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6640,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090743357851570176","occurred_at":"2026-08-21T10:10:35Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":6625,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090725692609138688","occurred_at":"2026-08-21T09:00:23Z","signature_id":"e20b12b46b2b1548a3a16720272c3402c384fc87cdfc8f7364b151aa7395d0a4","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090717616074133504","occurred_at":"2026-08-21T08:28:17Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090717583954153472","occurred_at":"2026-08-21T08:28:10Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090683781416488960","occurred_at":"2026-08-21T06:13:51Z","signature_id":"2c875be0e7e1b99f532d6815b3e15d00a56345749d48b8894ad74303128fd231","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090678628760162304","occurred_at":"2026-08-21T05:53:22Z","signature_id":"497befd8d2afd386b85a479e07ff0f9765dd27d9ab898de2a689c72a9da46088","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"19a00a393ae2b995155105e1565d2edf169aa75402e7aa5d511c8fb52a234e24","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090659366838472704","occurred_at":"2026-08-21T04:36:40Z","signature_id":"d2967305299147339686eaf4edf2881f0b317bdd1a1bb179903e51ff8ef771b4","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090649438694412288","occurred_at":"2026-08-21T03:57:23Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090647994339692544","occurred_at":"2026-08-21T03:51:38Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090619074294845440","occurred_at":"2026-08-21T01:56:43Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090612481398411264","occurred_at":"2026-08-21T01:30:28Z","signature_id":"e0137a945f1e2a54ce81a232b4041838395e6a1684d4fcae4f95e8d894658012","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090598725549297664","occurred_at":"2026-08-21T00:35:52Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6630,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090587456947621888","occurred_at":"2026-08-20T23:51:05Z","signature_id":"bb91ddee5e06429218a8675042f0d96a4e566f9e4db42274ff778c8f11f500fd","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090580629858029568","occurred_at":"2026-08-20T23:23:55Z","signature_id":"ffd256ac63b2ff6003a6525bcb93a7edcb7673cf0154ec5c0226a9384ccdeba5","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090577382321688576","occurred_at":"2026-08-20T23:10:58Z","signature_id":"976bbcff91a0b028f2b0d3552b44e9606b7e58c2c2207457e9aaf3cd66c93efd","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090576352783634432","occurred_at":"2026-08-20T23:06:58Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090575344808497152","occurred_at":"2026-08-20T23:02:57Z","signature_id":"bb91ddee5e06429218a8675042f0d96a4e566f9e4db42274ff778c8f11f500fd","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090566162734649344","occurred_at":"2026-08-20T22:26:26Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6630,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090555484766474240","occurred_at":"2026-08-20T21:44:02Z","signature_id":"bd2fe2f5d4a8ce19f94859a4e624c3c5667704e60cc19879ec4b40432574a576","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090554196767019008","occurred_at":"2026-08-20T21:38:55Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090553390097502208","occurred_at":"2026-08-20T21:35:43Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090549448319963136","occurred_at":"2026-08-20T21:20:03Z","signature_id":"e0137a945f1e2a54ce81a232b4041838395e6a1684d4fcae4f95e8d894658012","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090548196454436864","occurred_at":"2026-08-20T21:15:05Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6633/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090523260918697984","occurred_at":"2026-08-20T19:36:00Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6633,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6632/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090522418434019328","occurred_at":"2026-08-20T19:32:39Z","signature_id":"fcec3585b17892e01a8a8d3acce94709e69a2c001f4b79a074973e51b2eafdf9","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514791213830144","occurred_at":"2026-08-20T19:02:20Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"d2967305299147339686eaf4edf2881f0b317bdd1a1bb179903e51ff8ef771b4","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090501897923334144","occurred_at":"2026-08-20T18:10:59Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090489518846119936","occurred_at":"2026-08-20T17:21:54Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":6278,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090484744067223552","occurred_at":"2026-08-20T17:02:48Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6625/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090479903731355648","occurred_at":"2026-08-20T16:43:42Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6625,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090463536332410880","occurred_at":"2026-08-20T15:09:03Z","signature_id":"41651525e2aba85d110545005660d80238d17590a754aa3d1b78632adaaaf0ed","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090448552185565184","occurred_at":"2026-08-20T14:37:48Z","signature_id":"b7fbd103a195b66ac94bc7078697b03cb8d35c8e0fee824888587e1e72a89148","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090394899235999744","occurred_at":"2026-08-20T11:05:56Z","signature_id":"bb91ddee5e06429218a8675042f0d96a4e566f9e4db42274ff778c8f11f500fd","pr_number":6592,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090373033880457216","occurred_at":"2026-08-20T09:39:03Z","signature_id":"4fcebcb05bfc80f2d35d69227639a0e21e16fdc0d0a4461c0b08ead5dba75150","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090355263239360512","occurred_at":"2026-08-20T08:28:26Z","signature_id":"955efdfa565c09addf9d705f480891c18ea8c9dcafa7daf71cdc742a8e04bd87","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"de88c780ac57910f5dd0f3ea4c4fe8777e1ae26bc3f67f21242d59fc868296e7","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090344759288664064","occurred_at":"2026-08-20T07:46:41Z","signature_id":"d2967305299147339686eaf4edf2881f0b317bdd1a1bb179903e51ff8ef771b4","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090334446552092672","occurred_at":"2026-08-20T07:05:43Z","signature_id":"a9ecbd925234d950c6ec9a4c432a806852925b879c184fba9cd8c97ceceb05f5","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6616/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090329417594179584","occurred_at":"2026-08-20T06:45:44Z","signature_id":"e0137a945f1e2a54ce81a232b4041838395e6a1684d4fcae4f95e8d894658012","pr_number":6616,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090283973790928896","occurred_at":"2026-08-20T03:45:09Z","signature_id":"b998d1848e9e74b3cafa33bfa1cafbba26000bfe45b96ebc3b692b721613768d","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090266089802960896","occurred_at":"2026-08-20T02:34:05Z","signature_id":"bb91ddee5e06429218a8675042f0d96a4e566f9e4db42274ff778c8f11f500fd","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090253315500675072","occurred_at":"2026-08-20T01:43:20Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"521837880a2f6f77d1ffd5ab821dd3785b6f84bb96c295368a3577aaaf98e844","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090214858837463040","occurred_at":"2026-08-19T23:10:31Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6614,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090212568353214464","occurred_at":"2026-08-19T23:01:14Z","signature_id":"e0137a945f1e2a54ce81a232b4041838395e6a1684d4fcae4f95e8d894658012","pr_number":6588,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090212556621746176","occurred_at":"2026-08-19T23:01:09Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090206132864815104","occurred_at":"2026-08-19T22:35:50Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090198433062195200","occurred_at":"2026-08-19T22:05:11Z","signature_id":"f43d4b8968a344f7689d525224fffe656bfdbfeb15dfa483f233040306cfd422","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090192149080444928","occurred_at":"2026-08-19T21:40:16Z","signature_id":"c17108aafdd26274dff3fa12598513024038c96e6670180dc72078501ecefea5","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6572/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090182266104844288","occurred_at":"2026-08-19T21:01:00Z","signature_id":"bb91ddee5e06429218a8675042f0d96a4e566f9e4db42274ff778c8f11f500fd","pr_number":6572,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090179751976112128","occurred_at":"2026-08-19T20:51:01Z","signature_id":"e0026eefcc9d021330722912e2d8eb29e1d3fde3e81032401b2fd763208297bd","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090166116277030912","occurred_at":"2026-08-19T19:56:50Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090159046848942080","occurred_at":"2026-08-19T19:28:44Z","signature_id":"bb91ddee5e06429218a8675042f0d96a4e566f9e4db42274ff778c8f11f500fd","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090119905717260288","occurred_at":"2026-08-19T16:53:12Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090119424534122496","occurred_at":"2026-08-19T16:51:17Z","signature_id":"ac7c23c664839f4ba518b63f650cf0315a11f7d86a647ed599ba5ba0a729f061","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090105093117448192","occurred_at":"2026-08-19T15:54:21Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6592,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090100918157578240","occurred_at":"2026-08-19T15:37:04Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6426,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085989094526976","occurred_at":"2026-08-19T14:38:26Z","signature_id":"fcec3585b17892e01a8a8d3acce94709e69a2c001f4b79a074973e51b2eafdf9","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6600/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085087973478400","occurred_at":"2026-08-19T14:34:51Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6600,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090055453999370240","occurred_at":"2026-08-19T12:37:06Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089998936298229760","occurred_at":"2026-08-19T08:52:31Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6599,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089982500192391168","occurred_at":"2026-08-19T07:47:12Z","signature_id":"a179f310e31feef8931f82d5879b3d20bc9a6c8f6cae34f97512a9f6c979cd54","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089907805891858432","occurred_at":"2026-08-19T02:50:24Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6558/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089876846182141952","occurred_at":"2026-08-19T00:47:22Z","signature_id":"dec22c8d7025ad850ec3e80affbb82b1b1ebdec8a7deb9b908c0fcd23d44b728","pr_number":6558,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6593/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089864988096204800","occurred_at":"2026-08-19T00:00:06Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6593,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":71,"runs_affected":71,"failed_runs":71,"run_impact_percent":11.198738170347003,"anchor_occurrences":71,"after_last_push_count":16,"also_in":["INT","PROD","STG"],"trend_counts":[0,0,0,70,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-17"],"prior_runs_affected":74,"prior_last_seen_at":"2026-08-21T21:06:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":22},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":48},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:220]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000012048\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22, targetMinor=5.0) failed: clusterversion status.history has no version in target minor \"5.0\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:220]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000012048\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22, targetMinor=5.0) failed: clusterversion status.history has no version in target minor \"5.0\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:194]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00195b230\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:194]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00195b230\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:194]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001282120\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:194]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001282120\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"9022b451906b9a6a0432bfd8f6a4bba8e8dea2ae8057ca72799a51fd124f9bca","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090908436404375552","occurred_at":"2026-08-21T21:06:33Z","signature_id":"4090f06022a82484d544da5fc0f946bd246078a58a2fc784c8020ca944fe069f","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894768476262400","occurred_at":"2026-08-21T20:12:08Z","signature_id":"0382ce8a1155781d97bef6495e5c7a6669d7186d4d84f3bc2bf06b0a0d015de3","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894762604236800","occurred_at":"2026-08-21T20:12:07Z","signature_id":"ab796cc2f837d109fd6c7397a4be71f27775f755efc3f9b1ad40beb70f93595f","pr_number":6646,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894756736405504","occurred_at":"2026-08-21T20:12:05Z","signature_id":"2c1c634c357c5606e87b00bc23ab15eefacde5af4a249ffc1395fa0077cb9efc","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894675127832576","occurred_at":"2026-08-21T20:11:52Z","signature_id":"ccc09dd771e181c0dae77e9a7fd57d84a7a07fd1b75591cad8507db7ecb4c61f","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090884661910704128","occurred_at":"2026-08-21T19:32:04Z","signature_id":"ab0c49f3bf28254b3148cedfad87253207cd57a59f17c10afbf49b05b0290a36","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090882234021056512","occurred_at":"2026-08-21T19:22:25Z","signature_id":"922650986953b8202aefb7a4964bc0321d02b0ab4447ecdf89a383072d8c5f5b","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6593/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090879591731171328","occurred_at":"2026-08-21T19:11:50Z","signature_id":"2ddc98ca1c293c2673fbf12ebd368e974d9e10640b768c7a482963ef32c95ae1","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090878713473273856","occurred_at":"2026-08-21T19:08:16Z","signature_id":"4c4fa7fb1bf625b2d3247b2379b358428d0bce0e17e28d67498458083669e75b","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090877968120287232","occurred_at":"2026-08-21T19:04:55Z","signature_id":"bda765f4ccf539c4ee0129f74962705e6379bbba6aa02e9859330a3d2c5dc3e6","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090871776832655360","occurred_at":"2026-08-21T18:40:52Z","signature_id":"b1670c7da43304dc559d677b292763c03c30dda0db3d1248632ea3056d150ddb","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6650/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090866482933141504","occurred_at":"2026-08-21T18:19:50Z","signature_id":"19dafc7bc774815638556ca7b37fef1969115a880a613ecf7527e2301d85187a","pr_number":6650,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849047588900864","occurred_at":"2026-08-21T17:10:33Z","signature_id":"b8688968de3c5ef5b2db6bfa888b16997a3b71cf9d9183396c6c376709df7cc7","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849031038177280","occurred_at":"2026-08-21T17:10:29Z","signature_id":"ffece2232b540a63d0803ff30b0e3c8fbef0ab71e70b9d4c20af731d95bedc34","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849020749549568","occurred_at":"2026-08-21T17:10:27Z","signature_id":"c72c6838ad7c4bae62f8f4cd0430795160801a2135e286abad7a5f87c488245c","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090848752863547392","occurred_at":"2026-08-21T17:09:23Z","signature_id":"e823c669604f208a00298d07aa6dc273511165907708d46e387b703b047b5cb9","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090837373561606144","occurred_at":"2026-08-21T16:23:45Z","signature_id":"397a410f407a5210d3f56529fec7e6e07d0b098af14ba62c19445a400fae3187","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090837156841918464","occurred_at":"2026-08-21T16:23:18Z","signature_id":"1ef873971e878e4763e4ff1868eab81d4b41b24afe427ec15e91f0aa15eaacf7","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6647/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090836865526534144","occurred_at":"2026-08-21T16:22:00Z","signature_id":"b6aababddc1cea32acce54ad7f303b1d7e1bb032433f83a7ef1bab88320054a1","pr_number":6647,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090836156110344192","occurred_at":"2026-08-21T16:19:20Z","signature_id":"a3c5ac894514a4d5681ed7a6c47cdc78a663701ede7adcf20cd9e767c2555485","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090833818511478784","occurred_at":"2026-08-21T16:10:02Z","signature_id":"f41376e0fd047969cb02e607037104a05ba22266b1beaf1cb69eafa2b436d7a6","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090833794218070016","occurred_at":"2026-08-21T16:09:56Z","signature_id":"c95408ad8246dd7163fe031ec6185530c3613df10726090cedfabb8d55ae263d","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817798367350784","occurred_at":"2026-08-21T15:06:21Z","signature_id":"148240e213c49694b88bf4e5f3bc8b75f8e2f1924a548767dfb616662fd80005","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817534927310848","occurred_at":"2026-08-21T15:05:19Z","signature_id":"b3d8c5ac4a8dead5518d320568d761e00060b54e4d32594b44b568f24b398b97","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090811015728992256","occurred_at":"2026-08-21T14:31:41Z","signature_id":"a5825696e84694b99c030c756284c4e9a073e005bd400906c8cc01050aa12a6f","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"0ecd479311dc3e3983f7b5d80071fce53a706e25650eb18d08bdde3f73141495","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090805741328797696","occurred_at":"2026-08-21T14:18:28Z","signature_id":"9faef49dd63ed61c994ede57fd7dfae2df4d6be0db3c901d202045d895b077c3","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090804889495015424","occurred_at":"2026-08-21T14:15:05Z","signature_id":"bccfa8e61fa208ededbeb52840eb11c0c377b390c03f6641d031e037b5cd077f","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090803404556210176","occurred_at":"2026-08-21T14:09:11Z","signature_id":"135e19c4ac0830ce6714eb18154f43c293f34ec3cc7d98b682735da5fbd37f49","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090803249215967232","occurred_at":"2026-08-21T14:08:34Z","signature_id":"9da15eb1a64b2d2af52bf57fad31504e43850bec545a73e48fc7ccb8884a3f81","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090803211123298304","occurred_at":"2026-08-21T14:08:25Z","signature_id":"42e637ddf5c18937fb7b0f3287feddb3a95816fa6dfdc6a1a0b2c83598393fe2","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090803025537929216","occurred_at":"2026-08-21T14:07:41Z","signature_id":"a0632bb2f2060849353ff56a6005a99e05ef66ab605602be26920a61645415d2","pr_number":6646,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6647/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090794933425278976","occurred_at":"2026-08-21T13:35:19Z","signature_id":"75a337ac90d8611397e2787f50022df937374eb5fa76f2ca3d1f8c06fbd055a5","pr_number":6647,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6643/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090790935381151744","occurred_at":"2026-08-21T13:19:38Z","signature_id":"904aeee3492aac19dd9b67e842266a7947c6c5c1d8021b2c491d6d1badfbfbcd","pr_number":6643,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090784604393836544","occurred_at":"2026-08-21T12:54:29Z","signature_id":"060103ebf791bdc252e9fd7c3f2004368fc7294240ee45e80feaf2339b2c6c87","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090780869554868224","occurred_at":"2026-08-21T12:39:38Z","signature_id":"73a8bfe1e11251ff2d14478f901cefed6869821c0b0b81674cf8b8c4650624b6","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090779685267968000","occurred_at":"2026-08-21T12:34:56Z","signature_id":"812d71e6470288c16de9c3ff3c3d3de5bf15d2ed32e2e24c2f8078f4e7070425","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090779285378830336","occurred_at":"2026-08-21T12:33:21Z","signature_id":"96bbb2ae16d7c3fed47071662d0ea92d4f245f605dc4b6953361f7a32c5237e9","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090776820973572096","occurred_at":"2026-08-21T12:23:15Z","signature_id":"8e684d65c1dc3791016178d8e12d6c74b1706b047b2be7de1090ab37a3d8a6d8","pr_number":6640,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090773034322366464","occurred_at":"2026-08-21T12:08:30Z","signature_id":"7f2d125c3d23463139685011d35022cc1cc318fb866357defe713bd166e72735","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090770393882169344","occurred_at":"2026-08-21T11:58:01Z","signature_id":"eb1ffbf143a86bb0416996b0887acda8192476da3ccf504edbb9c3becda710b5","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090769277882732544","occurred_at":"2026-08-21T11:53:35Z","signature_id":"f9d437562fc5f93c2546a0272f5fefd7436a69a1b41db0828c34e8c779ed308a","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090768490683174912","occurred_at":"2026-08-21T11:50:27Z","signature_id":"f7c024e4f79c41c3ae7a7e53f4c95498df76592b592dfd6b8a171ee3536433db","pr_number":6646,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"8decc7f4d214fff8194d1174e836c4092ecd3b32e8d9bf66efc240fc275fd185","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090766839033368576","occurred_at":"2026-08-21T11:43:53Z","signature_id":"abf07e40a05be4b14049a34485a64c8a795ac2ad5eb7c759ffd4d80c9874faf5","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6643/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090757087574364160","occurred_at":"2026-08-21T11:05:08Z","signature_id":"d669bfaa1670fbad664b5ac73b7c79489bae68126c49cf2f1fba0eabc1c51316","pr_number":6643,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090753518062276608","occurred_at":"2026-08-21T10:50:57Z","signature_id":"6bfa7ec24001ab7354b3bd27d9ff458ec36b57a265693ecaec9a2df3f646d744","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090751316451135488","occurred_at":"2026-08-21T10:42:12Z","signature_id":"6dd2fb179ec22ce2728df7bd94aa1cce6d1a9cef31014de181ba8e933b51ba65","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090739392959221760","occurred_at":"2026-08-21T09:54:49Z","signature_id":"0a8114641b5c7afff27980ca306ad679af93fd5d4751a1dac3d983b42b867fda","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090725937179004928","occurred_at":"2026-08-21T09:00:54Z","signature_id":"59425a1054fbda560f28ca3e279dd01b90478c1e2854741e205910c434c21114","pr_number":6625,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090725692609138688","occurred_at":"2026-08-21T09:00:23Z","signature_id":"2141037c63ca4477ad055f5b74bf51e5e0c128c55d5a4ca3f343afe64247d3d9","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090720049605120000","occurred_at":"2026-08-21T08:37:58Z","signature_id":"a1d1494c1cd89e3934b69d63bf00641bcb31a00c1a245a6e4b92280f32d1b0c4","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090717616074133504","occurred_at":"2026-08-21T08:28:17Z","signature_id":"bd8a8f86b00d1c8cfb9c1cdaae4055ffba2dc8890c177c72364ba0ee601d38d2","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090717583954153472","occurred_at":"2026-08-21T08:28:10Z","signature_id":"681bf3b2806e864c549cf7fe8c2a3906ebe0676693b2b19614ceeb29b9cae8f9","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090711731759222784","occurred_at":"2026-08-21T08:04:32Z","signature_id":"16b25b47d2325fbbc2d1941dc8824c4a80b04e5f6973b34ca6ffea24a1207acc","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090693922316619776","occurred_at":"2026-08-21T06:54:08Z","signature_id":"64e25d112e0695e1d034cd57b56d23c9102d4d7807f92050e1757b8779c09444","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090688754837622784","occurred_at":"2026-08-21T06:33:36Z","signature_id":"0a94ed3abfcb10826ec265b69eee6c3d3f177e2b400b849a60c6c232870d25b2","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090683781416488960","occurred_at":"2026-08-21T06:13:51Z","signature_id":"3f8187d860ff81ce8e842c107134d560cc17b74b0d9a3d6f7bc6c288b56ff143","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090678628760162304","occurred_at":"2026-08-21T05:53:22Z","signature_id":"cd5e3a93de1c1b4a0f571db23612917f911fead22146e5cdddc4af2900037c22","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090677551943913472","occurred_at":"2026-08-21T05:49:05Z","signature_id":"b33468305bac6091a451daaf803ba20e051b55f2d05ad65d539c7a79fec82cdb","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"275c9b1cc9ab11aaa390a42bdd290c7343b52822e3c9e5fc88e7b2329f85da3e","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090660118717796352","occurred_at":"2026-08-21T04:39:49Z","signature_id":"de77568282fe39219f9c9d230c95485c7c2a386ecba55052652842309ae74587","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090659366838472704","occurred_at":"2026-08-21T04:36:40Z","signature_id":"17d1501457e51a3d2cc1966853d30f056c3c01a9b8dfa3fc989a9476679ca64c","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"ea6bd51c7d5e9d4edc59703a240cffeabbf9e75056df123880d04b0e8d5aa06a","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"2317d1519d946e5b068390bd95b67193bef53c958391cf5cfe1e06907335eb51","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090649438694412288","occurred_at":"2026-08-21T03:57:23Z","signature_id":"c4ec8921333fd1436f1e90888ca42212a45ef348fe432fce088bc6c75bb586b8","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090647994339692544","occurred_at":"2026-08-21T03:51:38Z","signature_id":"a89bb15d917c56506b65981e3e85aca7499e44b9e24eaaac137e1fcc0c0b4f38","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090646203308969984","occurred_at":"2026-08-21T03:44:31Z","signature_id":"d1e1dcdaf209cb4d50df4bb2b230ba25fd816f993dd504a0b70c2a90b8ac9196","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090644128227397632","occurred_at":"2026-08-21T03:36:17Z","signature_id":"2f7014bea9a6fd8030e9f98db125d9ce51d2f0c53cd554d31900751eb41c884b","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090643626764800000","occurred_at":"2026-08-21T03:34:17Z","signature_id":"241ca05585b9c433095e06f7ae32c935c81bc64b5e7402465f48ef00f091cf14","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":67,"runs_affected":31,"failed_runs":31,"run_impact_percent":4.889589905362776,"anchor_occurrences":67,"after_last_push_count":215,"also_in":["STG"],"trend_counts":[43,30,18,17,0,0,1,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":169,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d6a348\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-t87xzxghf54p\" to finish deleting: GET https://rp.j8620928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37cc526b-9ea6-4ae1-aecf-c9876a926ca3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37cc526b-9ea6-4ae1-aecf-c9876a926ca3\",\n      \"name\": \"37cc526b-9ea6-4ae1-aecf-c9876a926ca3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:09:21.722027891Z\",\n      \"endTime\": \"2026-08-25T19:33:29.709053496Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdpegmnfgfgem1607a6eukk8b4p953h still exists (deletion dispatched at 2026-08-25T19:09:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d6a348\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-t87xzxghf54p\" to finish deleting: GET https://rp.j8620928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37cc526b-9ea6-4ae1-aecf-c9876a926ca3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37cc526b-9ea6-4ae1-aecf-c9876a926ca3\",\n      \"name\": \"37cc526b-9ea6-4ae1-aecf-c9876a926ca3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:09:21.722027891Z\",\n      \"endTime\": \"2026-08-25T19:33:29.709053496Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdpegmnfgfgem1607a6eukk8b4p953h still exists (deletion dispatched at 2026-08-25T19:09:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e8af48\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-mb2sm98hvbm8\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n      \"name\": \"76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:14:43.826676557Z\",\n      \"endTime\": \"2026-08-24T15:38:50.051171369Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm25j05l3q2f8a1e83cjmn4lda8g still exists (deletion dispatched at 2026-08-24T15:14:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e8af48\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-mb2sm98hvbm8\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n      \"name\": \"76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:14:43.826676557Z\",\n      \"endTime\": \"2026-08-24T15:38:50.051171369Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm25j05l3q2f8a1e83cjmn4lda8g still exists (deletion dispatched at 2026-08-24T15:14:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000efa360\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-hzr77tbdkrwt\" to finish deleting: GET https://rp.j0088320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf6ae19-b7a5-40cd-806b-5002e0ccbaca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf6ae19-b7a5-40cd-806b-5002e0ccbaca\",\n      \"name\": \"9bf6ae19-b7a5-40cd-806b-5002e0ccbaca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T22:23:11.170362844Z\",\n      \"endTime\": \"2026-08-21T22:47:12.159444276Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb7tt2c2j1a6r0ch04519gisqcoillq still exists (deletion dispatched at 2026-08-21T22:23:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000efa360\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-hzr77tbdkrwt\" to finish deleting: GET https://rp.j0088320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf6ae19-b7a5-40cd-806b-5002e0ccbaca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf6ae19-b7a5-40cd-806b-5002e0ccbaca\",\n      \"name\": \"9bf6ae19-b7a5-40cd-806b-5002e0ccbaca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T22:23:11.170362844Z\",\n      \"endTime\": \"2026-08-21T22:47:12.159444276Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb7tt2c2j1a6r0ch04519gisqcoillq still exists (deletion dispatched at 2026-08-21T22:23:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314114058620928","occurred_at":"2026-08-25T18:12:12Z","signature_id":"f9759b9996e08511f5b632db38111c4fa4b734c76990458a0d0586bc1dc45bbb","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"53ff086f26d6ee86f70c417c33372b3bc3883205891aacb5f42319d457fb9cf5","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"bb2b90e4f29b12a3abf513339d37627508f6b4fcb179c3da34fa33dfad9d2de1","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"8f4a0db4076e1cf40b1861e6f93b93e2128a0306906fcdbdc7164a522fed9fcb","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"c57fa18006932cc4fa1a5c28791cae06cbdd74d294833ef29cd896320113cdf5","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894675127832576","occurred_at":"2026-08-21T20:11:52Z","signature_id":"23ec657957baf3f24205544e835e35c4734001a72a2c3076d95cb8519ffbd14d","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894675127832576","occurred_at":"2026-08-21T20:11:52Z","signature_id":"9deaefb6491c44a515e441c706ff3b3b31a8258d9f6b976bb9e6d307cce036a7","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"138c6eb6ec4cfb0f559faf8b27491e176a7e3689afd15778fe3c46693d7cd281","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"ef819eba9a8e82676e750c398e22e3fc87dc87851fbe6f06f07fe711d52b6a42","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"f7382c924eb5a3492d2ea34a9d0cd2081b3398e3a83bada4556de6ed77fabd24","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"0cc8318af89f3897857545e5af36e051cda23e38362afc2f37da5c1480303070","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"402189eabab67acb37888ca48022e0b174cbf93a51611d0579b67a44a25ed753","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"83e5dcc44bb56342cd157a1d9a58d7061fb36881f41e5de18f10c278c5865f9a","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"a4dbdef4b3f336d8893721cb7f8b34aae93f20865aeee833ebec5277bd6add9d","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090660118717796352","occurred_at":"2026-08-21T04:39:49Z","signature_id":"c122f69522e512ddd46dbe33faa5fae6273587778e6ee59c2db8a19c13119aef","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"1a32c1a95efe7a29e295ccd982d2b44cffbba78df2bd1aaf0fe98ce1680dc016","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"4cb7505b78171aefbb12d1f02e1c343144f3ebcfd268c6c0359c6fbf21922214","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"e4927c48eeabe34bc637b2643be12f8daeb52ec7c7626d29784709d3ee69d6f9","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090643626764800000","occurred_at":"2026-08-21T03:34:17Z","signature_id":"c6f251e281ff40fad947b6e76150b6760e590c03389601e036c5186f4a26607d","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090587456947621888","occurred_at":"2026-08-20T23:51:05Z","signature_id":"2a86e1a5bc401c03003c5845b248e027c44091ff5733e9efbd5b9a4bae152e77","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090587456947621888","occurred_at":"2026-08-20T23:51:05Z","signature_id":"e74f713f7a408119d04d974b97bfa35814e064db65c1703843c725496b2f8453","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090580629858029568","occurred_at":"2026-08-20T23:23:55Z","signature_id":"848724995a1498efc32129c35dffe5f7984c95f5e05ad1db60ad5c24c1e46924","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"6cbf440bc241ca9f30112587152f829d7ea81b8499f42d4cbdf631a896f8d037","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"6dcb427140d419621d24ab58c4f01fc95f92d9e6a6a74b94f3ccef1dc39e23bc","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"d28512cbbd0640af93827eeaf63af32a97f35479064f9bacc1353f4b29ddd213","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"c4469e6bfaef3e79efc3520cfb16a3b01f272605bc2e4ebf424fbbe45f9f6f58","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"efb76380c356925b35cc743b1695d2943cf8e8c4170a58cb685a9f26127b55d0","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090473213871001600","occurred_at":"2026-08-20T16:17:01Z","signature_id":"617f17096ddaad86b01334d98d8a379bc6dd44009a8565f420ca7b061da140c8","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090473213871001600","occurred_at":"2026-08-20T16:17:01Z","signature_id":"bd4d29c5320aea71d9f4b595ff9a6f126321aa2f9bec81d58bb337e616cb3347","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090443849187987456","occurred_at":"2026-08-20T14:20:26Z","signature_id":"7aa544050c69c6477ca85a4c3af70f1dd4a2e874e303c1b6b9968230f6fdbba1","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090373215313465344","occurred_at":"2026-08-20T09:39:46Z","signature_id":"70706e0e8a21dc34bb78771715c8487aa916955fd148bb428a78d36cc02e8913","pr_number":6589,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"386fa1316771c34bf0673e90f0051a18d6c5b4d7928e72e6319a9e2b346bec51","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"6cf80e4f74032eb95372cfafe9faa70d564caae28c956b32fc461ed70bfbbbe5","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"7b94f144d48f01b8a2dffdde46780cdeade986950e1c5cb7f10e8de5f6208290","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"080b1980af2148b254ebc364ec074b8b45a3913630fbee6ed39ba719bddb6412","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"c4ce75502913f8985263519e26b78cede5d045fc2b6518bb4355d5eb0f87c381","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090249792243372032","occurred_at":"2026-08-20T01:29:20Z","signature_id":"64b35d7107bf8bba32f177b18c07e632a8b85536f55028f9e8786e6cba992f6d","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"e56d745d1631f725a61b02fb97814b4427228a287c46c699fe0e8e754c2d3097","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090126361980047360","occurred_at":"2026-08-19T17:18:51Z","signature_id":"177b39dfe4594f7e6fb06a6f7750f916b44fd74d75da8566dfc2541c1b7da113","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090126361980047360","occurred_at":"2026-08-19T17:18:51Z","signature_id":"22ca9334c873ccba6a931cdcceec158397b54365ca7d2223fd061a1747a742d4","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090119905717260288","occurred_at":"2026-08-19T16:53:12Z","signature_id":"c3dcb5c0b950d390430a28a8362471db978e5fa8025bdacec6e0307a4e0ffa13","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090111171486027776","occurred_at":"2026-08-19T16:18:30Z","signature_id":"4963cb1cd863c9546d3acfa468246c7a65ea1896bf8c3d2e866fb6d92513767c","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090111171486027776","occurred_at":"2026-08-19T16:18:30Z","signature_id":"66b05bcdba707719a29ade3a38a592258173565a7afd218650cb24d16360c142","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090111171486027776","occurred_at":"2026-08-19T16:18:30Z","signature_id":"a67162d2dd9c478c5a34a2c5a352c6518959da807cd853356ac9ef9bbb9355f5","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087196852752384","occurred_at":"2026-08-19T14:43:14Z","signature_id":"2e68ce4465ab7b4784295bc0f537a0cc12ab0b9eb033768dfce3b9397fbabc37","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087196852752384","occurred_at":"2026-08-19T14:43:14Z","signature_id":"991fa55c1cbf281c8a6a21502a56f3cf0243c994a46b84c3c129c86d08500e94","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087196852752384","occurred_at":"2026-08-19T14:43:14Z","signature_id":"d1f00d98b96fc43cd8cd04fb3f710f5c58d8e5c745304cfa7bed7ca8f9cf798d","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085264507539456","occurred_at":"2026-08-19T14:35:33Z","signature_id":"c7e64cdff7fd089d3fd2dd3b4ae01e0ae9578c12c9f77faff280f06ac6241164","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085264507539456","occurred_at":"2026-08-19T14:35:33Z","signature_id":"d667f20e991a417af83dd002ae338e794d5ed667f5e3b1ee7b027b63a16e69a5","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"45c9a8bb121ddb135d411d85d961a4f1313b74d1f7b28d3f73e8aa3e30aa988b","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"a7de2a9cdec6164a3b14253e84d4f4dfe750c11eb696b14caf4efcef018b75d4","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"ea68d0fb40ccf06f806678a58f24d5aa4bd9e216ba376d8f98410f8788f2f8e3","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090072497390096384","occurred_at":"2026-08-19T13:44:49Z","signature_id":"2bfd6cd29638b751942a8e78dac53159ec41dd94cbed6747c22299cb4f4bb07c","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090072497390096384","occurred_at":"2026-08-19T13:44:49Z","signature_id":"4cab3b4d6ef0a8436b0f34860e699e4f385289e25548e1ad56907806d84b53fb","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090072497390096384","occurred_at":"2026-08-19T13:44:49Z","signature_id":"6ccf4df41b29745ca9abeed380bd011941d1644aea3e42ab4bd657f5c0fe34e8","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090072497390096384","occurred_at":"2026-08-19T13:44:49Z","signature_id":"ee9a66e8fa637968d82e9e83d22e2c61a0f2d297af276122f3694772e62d3ec0","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"0a0c2d83c66feb449a7f4f87715a6640117bc4728f1ca975a82944b2f9039dc5","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"e5b646e94208fbfa23f1fac70760633b23e949b9bde01b54e28ea0f33aa95610","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"0acfb37bffb4ca551a4e28feb3c3075bb4d22fc8ed7e9fb872066197036c0ac6","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"238018c5ad3f7c198e49aee43504d6edb675df266f6262bd7b5ac202c0734a6e","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"27310541c6ac41b2b15867277516029215eee68b577dd2c8cdd5def731cfa578","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"3a541056af9b027a6109e929bd67604b5073754ff890abd93acfaff50f02560f","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"4761349e386d913979b40fe0b4022048cffc3c3fdf92f8f9e42ba5d0cb06da16","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"508abeacf94e71536edd29c58a77ec4b70b98367a1cad9c7784924a018e77099","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"8e6bebd5be6cbfa0c7c75bcd150afeb11ef0a28af22bd5a0ad8ff061092869e7","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"9ab123f5de31a039a03a1f1e26d3e5d42a79d3aa49f1e86d7896c2263ebbbdb6","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"c8797ce0def20fde7cfb2ece808b9a6049c4d35b238a23cf6bd47744bb8684d5","pr_number":6602,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":57,"runs_affected":26,"failed_runs":26,"run_impact_percent":4.100946372239748,"anchor_occurrences":57,"after_last_push_count":108,"also_in":["PROD","STG"],"trend_counts":[3,2,0,27,5,0,4,19],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":164,"prior_last_seen_at":"2026-08-22T00:26:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000abe4c0\u003e: \n    failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-b7hpj2n7vp57\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6613b466-27ca-4523-a803-29e6469bc399\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6613b466-27ca-4523-a803-29e6469bc399\",\n      \"name\": \"6613b466-27ca-4523-a803-29e6469bc399\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:41.448636527Z\",\n      \"endTime\": \"2026-08-25T19:39:42.477039993Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000abe4c0\u003e: \n    failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-b7hpj2n7vp57\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6613b466-27ca-4523-a803-29e6469bc399\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6613b466-27ca-4523-a803-29e6469bc399\",\n      \"name\": \"6613b466-27ca-4523-a803-29e6469bc399\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:41.448636527Z\",\n      \"endTime\": \"2026-08-25T19:39:42.477039993Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-wwz9mh at version 4.22.11\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8e6c0\u003e: \n    failed to create HCP cluster np-skip-minor-wwz9mh: failed waiting for cluster=\"np-skip-minor-wwz9mh\" in resourcegroup=\"rg-np-skip-minor-wwz9mh-j759r47mqgx8\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af4a457c-c413-4047-bdb1-2c68cdac3a98\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af4a457c-c413-4047-bdb1-2c68cdac3a98\",\n      \"name\": \"af4a457c-c413-4047-bdb1-2c68cdac3a98\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:50.487713455Z\",\n      \"endTime\": \"2026-08-25T19:36:50.727476187Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-wwz9mh at version 4.22.11\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8e6c0\u003e: \n    failed to create HCP cluster np-skip-minor-wwz9mh: failed waiting for cluster=\"np-skip-minor-wwz9mh\" in resourcegroup=\"rg-np-skip-minor-wwz9mh-j759r47mqgx8\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af4a457c-c413-4047-bdb1-2c68cdac3a98\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af4a457c-c413-4047-bdb1-2c68cdac3a98\",\n      \"name\": \"af4a457c-c413-4047-bdb1-2c68cdac3a98\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:50.487713455Z\",\n      \"endTime\": \"2026-08-25T19:36:50.727476187Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-6nrgxcz8znbm/cluster-candidate-5-0-bz6rh7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001044860\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-bz6rh7: failed waiting for cluster=\"cluster-candidate-5-0-bz6rh7\" in resourcegroup=\"rg-candidate-5-0-6nrgxcz8znbm\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0e826bb-86ad-484d-940a-a69d7c037731\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0e826bb-86ad-484d-940a-a69d7c037731\",\n      \"name\": \"c0e826bb-86ad-484d-940a-a69d7c037731\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:17.696794179Z\",\n      \"endTime\": \"2026-08-25T16:40:18.610816463Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-6nrgxcz8znbm/cluster-candidate-5-0-bz6rh7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001044860\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-bz6rh7: failed waiting for cluster=\"cluster-candidate-5-0-bz6rh7\" in resourcegroup=\"rg-candidate-5-0-6nrgxcz8znbm\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0e826bb-86ad-484d-940a-a69d7c037731\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0e826bb-86ad-484d-940a-a69d7c037731\",\n      \"name\": \"c0e826bb-86ad-484d-940a-a69d7c037731\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:17.696794179Z\",\n      \"endTime\": \"2026-08-25T16:40:18.610816463Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"b0b6ea785c6adb804d2ab49b2e9c929c0658e46f9052aac84f220ac1f23f4ce1","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"e6e6771b32f676dac7dd06703c5d639fb506b0db753539a9e145282b66c90963","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"7de4e86735efd5a13296026c22a5cfa492755634aa30ae6440c336d9757f0cba","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"7afa6a3cb0a5460454f74e2a2b1999dd33e3f60965567464435dde991505b283","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092257335450800128","occurred_at":"2026-08-25T14:26:35Z","signature_id":"f0c15c8af4ed67bd702e14b46d290c6f294cf901457e9886c4b4f48671c2cecc","pr_number":6669,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","signature_id":"672eae11be59af76cbd1f197619054a7417629804d462d7487cf8b975b64fdd2","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","signature_id":"a62be0324e4af4aa551666eb60b0b9159ceb611d57868bbf7b768ffe258fbfbc","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","signature_id":"d5aeb36659e2fe6d42bc3c374436c19a2f23ca86b7d193e2d4e494e04f3f5eda","pr_number":6630,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092244166900715520","occurred_at":"2026-08-25T13:34:16Z","signature_id":"5310caabe4a3b378f21ce354c7c0c3aac76e1ac8d392c8d5eaecc0741c517da0","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"09a91b6d59e4fbee757881e9171ec763df5844bb16934da2b1be95a803e452f3","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092215603107467264","occurred_at":"2026-08-25T11:40:45Z","signature_id":"2aaa34c36208117bcee63f75dc471d9e9468caff13d1659c2583035ece83c2fb","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"b43fff9cd9bd036194d64006e2bedb42a5de47a08b4bb6a5327c7a329e3576da","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"ed7279f8680b41f53bf9dea3800ff6d209dab4158b53dc3aa29303a69736db57","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"befa32cb5bdc16812833dab6b4ef32700219a3216e83ebcad65e2edc52d10b6c","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"8cb8819d08e22597597cadf9ee8ec483916ec2e91ed7308d6b08f79ac3c67f38","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"a7bdda2906b7a00d3ea96fe2c15e55467a147dabf9881ad88f1b9c2804a5d6af","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"e006fc8de127b51c0b9862eaa26e5d9a844309286eb194166ccd19b3c55c024d","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146681486249984","occurred_at":"2026-08-25T07:06:53Z","signature_id":"0d69566e1be51af62861a571d03269f04a5db50727a3f7e9802a7d8a5eb2ef75","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092142908357480448","occurred_at":"2026-08-25T06:51:54Z","signature_id":"4afdddbae344462c995de2251551a735a47164d0bd4bc3714ba035dce8cb03a0","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"409a88ec07ceec5d40d5ae23f780d5c74cad626824811548eedaa155d1eba475","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"3605097c99e7a1623e2e3dd91c5b9641833e016d1c0c516d9b0275085f65863f","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"4de409ed2fffcc46ddb557100c947bf3b037028f8c648ac70391763c58835df0","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","signature_id":"2cc87e67c265ba4f74a9a313e8f38a6f54437455cae5243f5e1f2cc3556cdfc9","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090958673797976064","occurred_at":"2026-08-22T00:26:10Z","signature_id":"32e3e639e79da48278f94d8391d74ce4cb85a363903f3ef9711f2b4a60031ae5","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090958673797976064","occurred_at":"2026-08-22T00:26:10Z","signature_id":"ef641fb77f87b6886bc522fb57a3da8fe06dbc7d26494080a421abb9ad752057","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090958470403592192","occurred_at":"2026-08-22T00:25:22Z","signature_id":"a064f13405d704dd956551e506d55eae97dc3afdd6ceed198a730a6dc40f8e9a","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090954745601593344","occurred_at":"2026-08-22T00:10:34Z","signature_id":"90e64bea2b5cd9fda39e4d27f36261d3fc85d5ff17672231eeb0e009c5e68bd7","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090954745601593344","occurred_at":"2026-08-22T00:10:34Z","signature_id":"abb07177c1d013f32243889baee16622a26170afeea839f5d47b0654ffbf9ac2","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"0349ddcd04864267fea2a40c6ffd0f5389f34b53149c3626152112cae34b7449","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"0551f4776b518df224c09b5adbdb96739a175249d48cb4ac3fac68e653ba2784","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"0c33d2cf14048080901688f04f7f168fea8a7a6a51e4092f2878897e923597fe","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"0e1a706139de5aa4816eca677714434847c2c72fbb8ca3cdf11469904c0b488e","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"104b8359ad40ff28797d2cc44d09f2edb8fa2cea7a9596a7bde475000ff6418a","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"1e1bf8d1b89ceea99d469469baddd96086177c0cdc9195d3a71cbb63354e5d71","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"22969b4eb4fa35e9e8b87425aca1b945193cd797a33ff2424068183b2e6ab9d5","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"2b14f6adbf7b7b91812954e1a14265de8a301e9d5fe741ae9b10e7b9bb6acf09","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"2e21fe98b5b1c02d97edb490b4df36950ff6461c0fb4f46deff583fbba4d3c69","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"4771c1f60aef391df6d58cb19d51e18d1459e9cdd584c483d829a743260e5437","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"5765f0a35ed890055f1423af6fb083204d4880be33ff573010e5ce36cde4c153","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"578e27c8f27eed3fe0ee1c891072f72be7bb6bef2e19cd87df7a2fc2e6f926e7","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"5cacea6c38638a09cbc35d06a63928290b6a37ac1805cf72182801f515e5b20e","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"5dafea2e96443aebcae093f90aba801deaac09c85776b69fee7e3e9f98f2ae4e","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"7f15a6d4df92119c16c31a6389fc33139cb6399bdee2ed13425acfdb032fa99c","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"7fd612d767c2ea3564f764b73b42a11ca9eea2b84ecbe772fd21a9c566dfae7b","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"89aba46a0c865e129eed75a6b31c0b3bc5dc7f077becd5cb01904d6e859b0c22","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"8a4cfe4d036a329b0fd9efe82e95b5a8589a1ed16beb75cbefa82f94fc66963c","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"957b662b32c881f5058300aba160316b07ec41cf6cde6fb9d85df6dea5cd5603","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"9a63130286fb0ef860c5f8085bc0379c1e9f1367550fb2fd9bb136a90e890736","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"a034d5a2d2675bf1f04eb0cf418b007ad1dff484ef6e4352e995b9ee9d000949","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"cf1f5ad07e8fcc4b5ad4eabc673ebe14ec2d966a56dd2f43bf7d01e12685a642","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"d21bd18a17ffb4f907acdd9f461d092ef8af245bb3b161c6ca73eb7c3b3c5126","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"d5cfde077190f3c18ba68ca0d18e114870bd9d384119e2b5f40b6488f746be3e","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"dad7476bfcf96373dfa42e63321486c9e8ee70265a767ae521fcda72aef7cd4e","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"e28e93d7094aad43c209ccd557ce0e6a208211122c2f3b778170a8496e2a9423","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"edcef64fd444b5b8f7c061e41f4f70744ef8ed0ca26c4e9d7a061b5ef6aa7762","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6596/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090162950290542592","occurred_at":"2026-08-19T19:44:15Z","signature_id":"f2bf06ca84a5627688967789c6d03c70cf37e834531750d81991aaaf88fece21","pr_number":6596,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"ffdb59f5b7f9768099566982a694648158846401c53b2afb3f022b9d904fa4e5","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":49,"runs_affected":49,"failed_runs":49,"run_impact_percent":7.728706624605678,"anchor_occurrences":49,"after_last_push_count":10,"trend_counts":[0,0,0,41,0,0,3,5],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":87,"prior_last_seen_at":"2026-08-21T20:12:08Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":49}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T17:40:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j0166528-svc\", component=\"backend\", name=\"triggercontrolplaneupgrade\", severity=\"warning\"\n  Description: Backend controller workqueue triggercontrolplaneupgrade has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T16:39:54Z\n  Ended:       2026-08-25T17:06:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j0912128-svc\", component=\"backend\", name=\"clusterresourcescontroller\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresourcescontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T11:10:19Z\n  Ended:       2026-08-25T11:46:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j8697984-svc\", component=\"backend\", name=\"observemanagedresourcegroup\", severity=\"warning\"\n  Description: Backend controller workqueue observemanagedresourcegroup has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"417c675e722a3d2e9207698265ffad01510d51821854d76bfab92157497f8c02","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146691078623232","occurred_at":"2026-08-25T07:06:55Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092071627658366976","occurred_at":"2026-08-25T02:08:39Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092031971118026752","occurred_at":"2026-08-24T23:30:58Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091995273302642688","occurred_at":"2026-08-24T21:05:15Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894768476262400","occurred_at":"2026-08-21T20:12:08Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894756736405504","occurred_at":"2026-08-21T20:12:05Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894675127832576","occurred_at":"2026-08-21T20:11:52Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090878713473273856","occurred_at":"2026-08-21T19:08:16Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090877968120287232","occurred_at":"2026-08-21T19:04:55Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090871776832655360","occurred_at":"2026-08-21T18:40:52Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6650/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090866482933141504","occurred_at":"2026-08-21T18:19:50Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6650,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849047588900864","occurred_at":"2026-08-21T17:10:33Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849031038177280","occurred_at":"2026-08-21T17:10:29Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090837373561606144","occurred_at":"2026-08-21T16:23:45Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090837156841918464","occurred_at":"2026-08-21T16:23:18Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6647/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090836865526534144","occurred_at":"2026-08-21T16:22:00Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6647,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090836156110344192","occurred_at":"2026-08-21T16:19:20Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090833818511478784","occurred_at":"2026-08-21T16:10:02Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090833794218070016","occurred_at":"2026-08-21T16:09:56Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090804889495015424","occurred_at":"2026-08-21T14:15:05Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090803404556210176","occurred_at":"2026-08-21T14:09:11Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6647/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090794933425278976","occurred_at":"2026-08-21T13:35:19Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6647,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6643/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090790935381151744","occurred_at":"2026-08-21T13:19:38Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6643,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090784604393836544","occurred_at":"2026-08-21T12:54:29Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090780869554868224","occurred_at":"2026-08-21T12:39:38Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090773034322366464","occurred_at":"2026-08-21T12:08:30Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090768490683174912","occurred_at":"2026-08-21T11:50:27Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6646,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6643/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090757087574364160","occurred_at":"2026-08-21T11:05:08Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6643,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090753518062276608","occurred_at":"2026-08-21T10:50:57Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090725937179004928","occurred_at":"2026-08-21T09:00:54Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6625,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090725692609138688","occurred_at":"2026-08-21T09:00:23Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090717616074133504","occurred_at":"2026-08-21T08:28:17Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090717583954153472","occurred_at":"2026-08-21T08:28:10Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090711731759222784","occurred_at":"2026-08-21T08:04:32Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090693922316619776","occurred_at":"2026-08-21T06:54:08Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090688754837622784","occurred_at":"2026-08-21T06:33:36Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090683781416488960","occurred_at":"2026-08-21T06:13:51Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090677551943913472","occurred_at":"2026-08-21T05:49:05Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090660118717796352","occurred_at":"2026-08-21T04:39:49Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090659366838472704","occurred_at":"2026-08-21T04:36:40Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090644128227397632","occurred_at":"2026-08-21T03:36:17Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090643626764800000","occurred_at":"2026-08-21T03:34:17Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":49,"runs_affected":49,"failed_runs":49,"run_impact_percent":7.728706624605678,"anchor_occurrences":49,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,35,14],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":49,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":49}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0006c4270\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[08:07:56.002]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:43.2162872Z);\\nlet _endTime = datetime(2026-08-25T08:07:43.2162878Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T08:07:55.9842316Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f\\\",\\r\\n            \\\"activityId\\\": \\\"93240ce7-344d-4b28-a6d5-bc7c0d41eeab\\\",\\r\\n            \\\"subActivityId\\\": \\\"daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"2d5a9b37-61e4-4d07-80a4-b16788917aef\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f ARID=93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.Http.CallContext/93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.ExecuteQuery/2d5a9b37-61e4-4d07-80a4-b16788917aef \\u003e DN.FE.ExecuteQuery/daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[08:07:56.054]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:43.2162872Z);\\nlet _endTime = datetime(2026-08-25T08:07:43.2162878Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T08:07:55.9842316Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f\\\",\\r\\n            \\\"activityId\\\": \\\"93240ce7-344d-4b28-a6d5-bc7c0d41eeab\\\",\\r\\n            \\\"subActivityId\\\": \\\"daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"2d5a9b37-61e4-4d07-80a4-b16788917aef\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f ARID=93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.Http.CallContext/93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.ExecuteQuery/2d5a9b37-61e4-4d07-80a4-b16788917aef \\u003e DN.FE.ExecuteQuery/daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[08:08:04.153]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:56.1417247Z);\\nlet _endTime = datetime(2026-08-25T08:07:56.1417253Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T08:08:04.1346416Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42\\\",\\r\\n            \\\"activityId\\\": \\\"5170d421-679a-4a48-b29b-9a996c74dde4\\\",\\r\\n            \\\"subActivityId\\\": \\\"d37306ab-e852-439a-bb92-4f7c817288f6\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"36329843-81dc-4dc4-aa28-c56ee7342c50\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42 ARID=5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.Http.CallContext/5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.ExecuteQuery/36329843-81dc-4dc4-aa28-c56ee7342c50 \\u003e DN.FE.ExecuteQuery/d37306ab-e852-439a-bb92-4f7c817288f6)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[08:08:04.206]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:56.1417247Z);\\nlet _endTime = datetime(2026-08-25T08:07:56.1417253Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T08:08:04.1346416Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42\\\",\\r\\n            \\\"activityId\\\": \\\"5170d421-679a-4a48-b29b-9a996c74dde4\\\",\\r\\n            \\\"subActivityId\\\": \\\"d37306ab-e852-439a-bb92-4f7c817288f6\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"36329843-81dc-4dc4-aa28-c56ee7342c50\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42 ARID=5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.Http.CallContext/5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.ExecuteQuery/36329843-81dc-4dc4-aa28-c56ee7342c50 \\u003e DN.FE.ExecuteQuery/d37306ab-e852-439a-bb92-4f7c817288f6)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[08:08:15.998]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:08:07.6782331Z);\\nlet _endTime = datetime(2026-08-25T08:08:07.6782337Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T08:08:15.9798710Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b\\\",\\r\\n            \\\"activityId\\\": \\\"c0534e85-0700-4932-b07d-9b9e7af63995\\\",\\r\\n            \\\"subActivityId\\\": \\\"abf8d736-f9c7-4683-a287-d5c9f53d5b71\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"54a6cff9-747f-4ac5-8f21-5804ee2d36e8\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b ARID=c0534e85-0700-4932-b07d-9b9e7af63995 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/54a6cff9-747f-4ac5-8f21-5804ee2d36e8 \\u003e DN.FE.ExecuteQuery/abf8d736-f9c7-4683-a287-d5c9f53d5b71)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[08:08:16.102]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:08:07.6782331Z);\\nlet _endTime = datetime(2026-08-25T08:08:07.6782337Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T08:08:15.9798710Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b\\\",\\r\\n            \\\"activityId\\\": \\\"c0534e85-0700-4932-b07d-9b9e7af63995\\\",\\r\\n            \\\"subActivityId\\\": \\\"abf8d736-f9c7-4683-a287-d5c9f53d5b71\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"54a6cff9-747f-4ac5-8f21-5804ee2d36e8\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b ARID=c0534e85-0700-4932-b07d-9b9e7af63995 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/54a6cff9-747f-4ac5-8f21-5804ee2d36e8 \\u003e DN.FE.ExecuteQuery/abf8d736-f9c7-4683-a287-d5c9f53d5b71)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0006c4270\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[08:07:56.002]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:43.2162872Z);\\nlet _endTime = datetime(2026-08-25T08:07:43.2162878Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T08:07:55.9842316Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f\\\",\\r\\n            \\\"activityId\\\": \\\"93240ce7-344d-4b28-a6d5-bc7c0d41eeab\\\",\\r\\n            \\\"subActivityId\\\": \\\"daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"2d5a9b37-61e4-4d07-80a4-b16788917aef\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f ARID=93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.Http.CallContext/93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.ExecuteQuery/2d5a9b37-61e4-4d07-80a4-b16788917aef \\u003e DN.FE.ExecuteQuery/daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[08:07:56.054]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:43.2162872Z);\\nlet _endTime = datetime(2026-08-25T08:07:43.2162878Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T08:07:55.9842316Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f\\\",\\r\\n            \\\"activityId\\\": \\\"93240ce7-344d-4b28-a6d5-bc7c0d41eeab\\\",\\r\\n            \\\"subActivityId\\\": \\\"daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"2d5a9b37-61e4-4d07-80a4-b16788917aef\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f956ea66-fdc4-476d-90bd-06769a04a23f ARID=93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.Http.CallContext/93240ce7-344d-4b28-a6d5-bc7c0d41eeab \\u003e GW.ExecuteQuery/2d5a9b37-61e4-4d07-80a4-b16788917aef \\u003e DN.FE.ExecuteQuery/daa37bf3-b9a2-4c43-9bab-6f0f919d8ffc)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[08:08:04.153]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:56.1417247Z);\\nlet _endTime = datetime(2026-08-25T08:07:56.1417253Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T08:08:04.1346416Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42\\\",\\r\\n            \\\"activityId\\\": \\\"5170d421-679a-4a48-b29b-9a996c74dde4\\\",\\r\\n            \\\"subActivityId\\\": \\\"d37306ab-e852-439a-bb92-4f7c817288f6\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"36329843-81dc-4dc4-aa28-c56ee7342c50\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42 ARID=5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.Http.CallContext/5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.ExecuteQuery/36329843-81dc-4dc4-aa28-c56ee7342c50 \\u003e DN.FE.ExecuteQuery/d37306ab-e852-439a-bb92-4f7c817288f6)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[08:08:04.206]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:07:56.1417247Z);\\nlet _endTime = datetime(2026-08-25T08:07:56.1417253Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T08:08:04.1346416Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42\\\",\\r\\n            \\\"activityId\\\": \\\"5170d421-679a-4a48-b29b-9a996c74dde4\\\",\\r\\n            \\\"subActivityId\\\": \\\"d37306ab-e852-439a-bb92-4f7c817288f6\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"36329843-81dc-4dc4-aa28-c56ee7342c50\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;02b7ec99-eb74-4396-80fc-e477f9f18b42 ARID=5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.Http.CallContext/5170d421-679a-4a48-b29b-9a996c74dde4 \\u003e GW.ExecuteQuery/36329843-81dc-4dc4-aa28-c56ee7342c50 \\u003e DN.FE.ExecuteQuery/d37306ab-e852-439a-bb92-4f7c817288f6)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[08:08:15.998]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:08:07.6782331Z);\\nlet _endTime = datetime(2026-08-25T08:08:07.6782337Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T08:08:15.9798710Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b\\\",\\r\\n            \\\"activityId\\\": \\\"c0534e85-0700-4932-b07d-9b9e7af63995\\\",\\r\\n            \\\"subActivityId\\\": \\\"abf8d736-f9c7-4683-a287-d5c9f53d5b71\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"54a6cff9-747f-4ac5-8f21-5804ee2d36e8\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b ARID=c0534e85-0700-4932-b07d-9b9e7af63995 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/54a6cff9-747f-4ac5-8f21-5804ee2d36e8 \\u003e DN.FE.ExecuteQuery/abf8d736-f9c7-4683-a287-d5c9f53d5b71)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[08:08:16.102]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T08:08:07.6782331Z);\\nlet _endTime = datetime(2026-08-25T08:08:07.6782337Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-nc7qs5pbjx25'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T08:08:15.9798710Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b\\\",\\r\\n            \\\"activityId\\\": \\\"c0534e85-0700-4932-b07d-9b9e7af63995\\\",\\r\\n            \\\"subActivityId\\\": \\\"abf8d736-f9c7-4683-a287-d5c9f53d5b71\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"54a6cff9-747f-4ac5-8f21-5804ee2d36e8\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a2ee4734-5cfc-4e05-bbc1-57b733b1306b ARID=c0534e85-0700-4932-b07d-9b9e7af63995 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f52d0643-2180-4672-941a-2aa83b28c631 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/54a6cff9-747f-4ac5-8f21-5804ee2d36e8 \\u003e DN.FE.ExecuteQuery/abf8d736-f9c7-4683-a287-d5c9f53d5b71)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000812850\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[07:36:48.911]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:45.8465920Z);\\nlet _endTime = datetime(2026-08-25T07:36:45.8465927Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T07:36:48.8924657Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1\\\",\\r\\n            \\\"activityId\\\": \\\"2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672\\\",\\r\\n            \\\"subActivityId\\\": \\\"403b7745-bbff-4c9f-929d-476fbd66118e\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"e88d573b-aeb8-4a17-9fca-c87f04c6cdd9\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1 ARID=2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e88d573b-aeb8-4a17-9fca-c87f04c6cdd9 \\u003e DN.FE.ExecuteQuery/403b7745-bbff-4c9f-929d-476fbd66118e)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[07:36:48.962]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:45.8465920Z);\\nlet _endTime = datetime(2026-08-25T07:36:45.8465927Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T07:36:48.8924657Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1\\\",\\r\\n            \\\"activityId\\\": \\\"2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672\\\",\\r\\n            \\\"subActivityId\\\": \\\"403b7745-bbff-4c9f-929d-476fbd66118e\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"e88d573b-aeb8-4a17-9fca-c87f04c6cdd9\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1 ARID=2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e88d573b-aeb8-4a17-9fca-c87f04c6cdd9 \\u003e DN.FE.ExecuteQuery/403b7745-bbff-4c9f-929d-476fbd66118e)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[07:36:51.855]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:49.0279421Z);\\nlet _endTime = datetime(2026-08-25T07:36:49.0279424Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T07:36:51.8360877Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b\\\",\\r\\n            \\\"activityId\\\": \\\"f4d03f54-2298-4d29-b8e3-9d984c0bd2b9\\\",\\r\\n            \\\"subActivityId\\\": \\\"60e487db-9b15-48f2-a301-b66685f17ff4\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"4dc8f887-d2c7-4ef0-ad29-017a6723ebe4\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b ARID=f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.Http.CallContext/f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.ExecuteQuery/4dc8f887-d2c7-4ef0-ad29-017a6723ebe4 \\u003e DN.FE.ExecuteQuery/60e487db-9b15-48f2-a301-b66685f17ff4)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[07:36:51.908]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:49.0279421Z);\\nlet _endTime = datetime(2026-08-25T07:36:49.0279424Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T07:36:51.8360877Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b\\\",\\r\\n            \\\"activityId\\\": \\\"f4d03f54-2298-4d29-b8e3-9d984c0bd2b9\\\",\\r\\n            \\\"subActivityId\\\": \\\"60e487db-9b15-48f2-a301-b66685f17ff4\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"4dc8f887-d2c7-4ef0-ad29-017a6723ebe4\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b ARID=f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.Http.CallContext/f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.ExecuteQuery/4dc8f887-d2c7-4ef0-ad29-017a6723ebe4 \\u003e DN.FE.ExecuteQuery/60e487db-9b15-48f2-a301-b66685f17ff4)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[07:36:58.311]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:55.2926733Z);\\nlet _endTime = datetime(2026-08-25T07:36:55.2926740Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T07:36:58.2926720Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958\\\",\\r\\n            \\\"activityId\\\": \\\"5168b874-e371-442d-96d4-634b404e01b7\\\",\\r\\n            \\\"subActivityId\\\": \\\"a4dec94c-d4be-425e-8443-3e3dba0c4a46\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"33d6b2a1-bd08-472a-87a5-98f6eafd09fb\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958 ARID=5168b874-e371-442d-96d4-634b404e01b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/33d6b2a1-bd08-472a-87a5-98f6eafd09fb \\u003e DN.FE.ExecuteQuery/a4dec94c-d4be-425e-8443-3e3dba0c4a46)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[07:36:58.405]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:55.2926733Z);\\nlet _endTime = datetime(2026-08-25T07:36:55.2926740Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T07:36:58.2926720Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958\\\",\\r\\n            \\\"activityId\\\": \\\"5168b874-e371-442d-96d4-634b404e01b7\\\",\\r\\n            \\\"subActivityId\\\": \\\"a4dec94c-d4be-425e-8443-3e3dba0c4a46\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"33d6b2a1-bd08-472a-87a5-98f6eafd09fb\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958 ARID=5168b874-e371-442d-96d4-634b404e01b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/33d6b2a1-bd08-472a-87a5-98f6eafd09fb \\u003e DN.FE.ExecuteQuery/a4dec94c-d4be-425e-8443-3e3dba0c4a46)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000812850\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[07:36:48.911]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:45.8465920Z);\\nlet _endTime = datetime(2026-08-25T07:36:45.8465927Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T07:36:48.8924657Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1\\\",\\r\\n            \\\"activityId\\\": \\\"2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672\\\",\\r\\n            \\\"subActivityId\\\": \\\"403b7745-bbff-4c9f-929d-476fbd66118e\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"e88d573b-aeb8-4a17-9fca-c87f04c6cdd9\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1 ARID=2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e88d573b-aeb8-4a17-9fca-c87f04c6cdd9 \\u003e DN.FE.ExecuteQuery/403b7745-bbff-4c9f-929d-476fbd66118e)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[07:36:48.962]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:45.8465920Z);\\nlet _endTime = datetime(2026-08-25T07:36:45.8465927Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T07:36:48.8924657Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1\\\",\\r\\n            \\\"activityId\\\": \\\"2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672\\\",\\r\\n            \\\"subActivityId\\\": \\\"403b7745-bbff-4c9f-929d-476fbd66118e\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"e88d573b-aeb8-4a17-9fca-c87f04c6cdd9\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;bbdb89c0-4fec-404c-bdd6-6d818f1cded1 ARID=2bf9b4f0-5c94-4ac1-bcf2-b176b70ad672 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/867d2a47-dbd4-435d-8c9b-766c6c523e4e \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e88d573b-aeb8-4a17-9fca-c87f04c6cdd9 \\u003e DN.FE.ExecuteQuery/403b7745-bbff-4c9f-929d-476fbd66118e)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[07:36:51.855]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:49.0279421Z);\\nlet _endTime = datetime(2026-08-25T07:36:49.0279424Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T07:36:51.8360877Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b\\\",\\r\\n            \\\"activityId\\\": \\\"f4d03f54-2298-4d29-b8e3-9d984c0bd2b9\\\",\\r\\n            \\\"subActivityId\\\": \\\"60e487db-9b15-48f2-a301-b66685f17ff4\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"4dc8f887-d2c7-4ef0-ad29-017a6723ebe4\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b ARID=f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.Http.CallContext/f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.ExecuteQuery/4dc8f887-d2c7-4ef0-ad29-017a6723ebe4 \\u003e DN.FE.ExecuteQuery/60e487db-9b15-48f2-a301-b66685f17ff4)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[07:36:51.908]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:49.0279421Z);\\nlet _endTime = datetime(2026-08-25T07:36:49.0279424Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T07:36:51.8360877Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b\\\",\\r\\n            \\\"activityId\\\": \\\"f4d03f54-2298-4d29-b8e3-9d984c0bd2b9\\\",\\r\\n            \\\"subActivityId\\\": \\\"60e487db-9b15-48f2-a301-b66685f17ff4\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"4dc8f887-d2c7-4ef0-ad29-017a6723ebe4\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d44a550b-983d-459e-b914-0653df2e0e3b ARID=f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.Http.CallContext/f4d03f54-2298-4d29-b8e3-9d984c0bd2b9 \\u003e GW.ExecuteQuery/4dc8f887-d2c7-4ef0-ad29-017a6723ebe4 \\u003e DN.FE.ExecuteQuery/60e487db-9b15-48f2-a301-b66685f17ff4)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[07:36:58.311]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:55.2926733Z);\\nlet _endTime = datetime(2026-08-25T07:36:55.2926740Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T07:36:58.2926720Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958\\\",\\r\\n            \\\"activityId\\\": \\\"5168b874-e371-442d-96d4-634b404e01b7\\\",\\r\\n            \\\"subActivityId\\\": \\\"a4dec94c-d4be-425e-8443-3e3dba0c4a46\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"33d6b2a1-bd08-472a-87a5-98f6eafd09fb\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958 ARID=5168b874-e371-442d-96d4-634b404e01b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/33d6b2a1-bd08-472a-87a5-98f6eafd09fb \\u003e DN.FE.ExecuteQuery/a4dec94c-d4be-425e-8443-3e3dba0c4a46)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[07:36:58.405]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T07:36:55.2926733Z);\\nlet _endTime = datetime(2026-08-25T07:36:55.2926740Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-vsdwgwkgtdjp'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T07:36:58.2926720Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958\\\",\\r\\n            \\\"activityId\\\": \\\"5168b874-e371-442d-96d4-634b404e01b7\\\",\\r\\n            \\\"subActivityId\\\": \\\"a4dec94c-d4be-425e-8443-3e3dba0c4a46\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"33d6b2a1-bd08-472a-87a5-98f6eafd09fb\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;da67d458-be70-4b7e-b6dd-55d1adf76958 ARID=5168b874-e371-442d-96d4-634b404e01b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d6496d65-ab35-4ce7-ab25-0229e9045b36 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/33d6b2a1-bd08-472a-87a5-98f6eafd09fb \\u003e DN.FE.ExecuteQuery/a4dec94c-d4be-425e-8443-3e3dba0c4a46)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000a22250\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[06:58:42.889]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:39.6001491Z);\\nlet _endTime = datetime(2026-08-25T06:58:39.6001496Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-bm9448kb54lr'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T06:58:42.8701809Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c\\\",\\r\\n            \\\"activityId\\\": \\\"50929d0b-6732-4e31-a429-a529a01fac9c\\\",\\r\\n            \\\"subActivityId\\\": \\\"29a9a3a1-3134-4073-9f9f-167f7be9ed6d\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"9e96b72c-b3c0-4907-ba16-c5f840899e89\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c ARID=50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.Http.CallContext/50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.ExecuteQuery/9e96b72c-b3c0-4907-ba16-c5f840899e89 \\u003e DN.FE.ExecuteQuery/29a9a3a1-3134-4073-9f9f-167f7be9ed6d)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[06:58:42.938]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:39.6001491Z);\\nlet _endTime = datetime(2026-08-25T06:58:39.6001496Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-bm9448kb54lr'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T06:58:42.8701809Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c\\\",\\r\\n            \\\"activityId\\\": \\\"50929d0b-6732-4e31-a429-a529a01fac9c\\\",\\r\\n            \\\"subActivityId\\\": \\\"29a9a3a1-3134-4073-9f9f-167f7be9ed6d\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"9e96b72c-b3c0-4907-ba16-c5f840899e89\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c ARID=50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.Http.CallContext/50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.ExecuteQuery/9e96b72c-b3c0-4907-ba16-c5f840899e89 \\u003e DN.FE.ExecuteQuery/29a9a3a1-3134-4073-9f9f-167f7be9ed6d)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[06:58:46.438]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:43.0033907Z);\\nlet _endTime = datetime(2026-08-25T06:58:43.0033913Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-bm9448kb54lr'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T06:58:46.4202164Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;2b381053-8eff-40bc-996d-50543376f61a\\\",\\r\\n            \\\"activityId\\\": \\\"4fca92f9-d3e2-4fa6-81c9-a9fc8513a713\\\",\\r\\n            \\\"subActivityId\\\": \\\"3821d24b-5b2c-40a4-bcae-6560c92f6b97\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"48e1149d-7cf7-4df9-9a59-15c62ac82ce9\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2b381053-8eff-40bc-996d-50543376f61a ARID=4fca92f9-d3e2-4fa6-81c9-a9fc8513a713 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/48e1149d-7cf7-4df9-9a59-15c62ac82ce9 \\u003e DN.FE.ExecuteQuery/3821d24b-5b2c-40a4-bcae-6560c92f6b97)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[06:58:46.487]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:43.0033907Z);\\nlet _endTime = datetime(2026-08-25T06:58:43.0033913Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-bm9448kb54lr'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T06:58:46.4202164Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;2b381053-8eff-40bc-996d-50543376f61a\\\",\\r\\n            \\\"activityId\\\": \\\"4fca92f9-d3e2-4fa6-81c9-a9fc8513a713\\\",\\r\\n            \\\"subActivityId\\\": \\\"3821d24b-5b2c-40a4-bcae-6560c92f6b97\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"48e1149d-7cf7-4df9-9a59-15c62ac82ce9\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2b381053-8eff-40bc-996d-50543376f61a ARID=4fca92f9-d3e2-4fa6-81c9-a9fc8513a713 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/48e1149d-7cf7-4df9-9a59-15c62ac82ce9 \\u003e DN.FE.ExecuteQuery/3821d24b-5b2c-40a4-bcae-6560c92f6b97)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[06:58:53.091]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:49.9195659Z);\\nlet _endTime = datetime(2026-08-25T06:58:49.9195666Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-bm9448kb54lr'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T06:58:53.0733729Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf\\\",\\r\\n            \\\"activityId\\\": \\\"8f036878-c750-44d8-94c2-d0ffa48c5023\\\",\\r\\n            \\\"subActivityId\\\": \\\"f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"ecd293b9-988c-49f9-988b-2c0d898fd0bb\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf ARID=8f036878-c750-44d8-94c2-d0ffa48c5023 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ecd293b9-988c-49f9-988b-2c0d898fd0bb \\u003e DN.FE.ExecuteQuery/f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[06:58:53.185]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:49.9195659Z);\\nlet _endTime = datetime(2026-08-25T06:58:49.9195666Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-bm9448kb54lr'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T06:58:53.0733729Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf\\\",\\r\\n            \\\"activityId\\\": \\\"8f036878-c750-44d8-94c2-d0ffa48c5023\\\",\\r\\n            \\\"subActivityId\\\": \\\"f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"ecd293b9-988c-49f9-988b-2c0d898fd0bb\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf ARID=8f036878-c750-44d8-94c2-d0ffa48c5023 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ecd293b9-988c-49f9-988b-2c0d898fd0bb \\u003e DN.FE.ExecuteQuery/f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000a22250\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[06:58:42.889]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:39.6001491Z);\\nlet _endTime = datetime(2026-08-25T06:58:39.6001496Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-bm9448kb54lr'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T06:58:42.8701809Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c\\\",\\r\\n            \\\"activityId\\\": \\\"50929d0b-6732-4e31-a429-a529a01fac9c\\\",\\r\\n            \\\"subActivityId\\\": \\\"29a9a3a1-3134-4073-9f9f-167f7be9ed6d\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"9e96b72c-b3c0-4907-ba16-c5f840899e89\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c ARID=50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.Http.CallContext/50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.ExecuteQuery/9e96b72c-b3c0-4907-ba16-c5f840899e89 \\u003e DN.FE.ExecuteQuery/29a9a3a1-3134-4073-9f9f-167f7be9ed6d)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[06:58:42.938]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:39.6001491Z);\\nlet _endTime = datetime(2026-08-25T06:58:39.6001496Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-bm9448kb54lr'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T06:58:42.8701809Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c\\\",\\r\\n            \\\"activityId\\\": \\\"50929d0b-6732-4e31-a429-a529a01fac9c\\\",\\r\\n            \\\"subActivityId\\\": \\\"29a9a3a1-3134-4073-9f9f-167f7be9ed6d\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"9e96b72c-b3c0-4907-ba16-c5f840899e89\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;54e3445c-767b-4a53-ab5e-6a8cb12abe5c ARID=50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.Http.CallContext/50929d0b-6732-4e31-a429-a529a01fac9c \\u003e GW.ExecuteQuery/9e96b72c-b3c0-4907-ba16-c5f840899e89 \\u003e DN.FE.ExecuteQuery/29a9a3a1-3134-4073-9f9f-167f7be9ed6d)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[06:58:46.438]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:43.0033907Z);\\nlet _endTime = datetime(2026-08-25T06:58:43.0033913Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-bm9448kb54lr'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T06:58:46.4202164Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;2b381053-8eff-40bc-996d-50543376f61a\\\",\\r\\n            \\\"activityId\\\": \\\"4fca92f9-d3e2-4fa6-81c9-a9fc8513a713\\\",\\r\\n            \\\"subActivityId\\\": \\\"3821d24b-5b2c-40a4-bcae-6560c92f6b97\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"48e1149d-7cf7-4df9-9a59-15c62ac82ce9\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2b381053-8eff-40bc-996d-50543376f61a ARID=4fca92f9-d3e2-4fa6-81c9-a9fc8513a713 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/48e1149d-7cf7-4df9-9a59-15c62ac82ce9 \\u003e DN.FE.ExecuteQuery/3821d24b-5b2c-40a4-bcae-6560c92f6b97)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[06:58:46.487]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:43.0033907Z);\\nlet _endTime = datetime(2026-08-25T06:58:43.0033913Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-bm9448kb54lr'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T06:58:46.4202164Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;2b381053-8eff-40bc-996d-50543376f61a\\\",\\r\\n            \\\"activityId\\\": \\\"4fca92f9-d3e2-4fa6-81c9-a9fc8513a713\\\",\\r\\n            \\\"subActivityId\\\": \\\"3821d24b-5b2c-40a4-bcae-6560c92f6b97\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"48e1149d-7cf7-4df9-9a59-15c62ac82ce9\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2b381053-8eff-40bc-996d-50543376f61a ARID=4fca92f9-d3e2-4fa6-81c9-a9fc8513a713 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/431aa304-259b-4063-8333-d98d7ccf1795 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/48e1149d-7cf7-4df9-9a59-15c62ac82ce9 \\u003e DN.FE.ExecuteQuery/3821d24b-5b2c-40a4-bcae-6560c92f6b97)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[06:58:53.091]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:49.9195659Z);\\nlet _endTime = datetime(2026-08-25T06:58:49.9195666Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-bm9448kb54lr'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T06:58:53.0733729Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf\\\",\\r\\n            \\\"activityId\\\": \\\"8f036878-c750-44d8-94c2-d0ffa48c5023\\\",\\r\\n            \\\"subActivityId\\\": \\\"f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"ecd293b9-988c-49f9-988b-2c0d898fd0bb\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf ARID=8f036878-c750-44d8-94c2-d0ffa48c5023 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ecd293b9-988c-49f9-988b-2c0d898fd0bb \\u003e DN.FE.ExecuteQuery/f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[06:58:53.185]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T06:58:49.9195659Z);\\nlet _endTime = datetime(2026-08-25T06:58:49.9195666Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-bm9448kb54lr'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T06:58:53.0733729Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf\\\",\\r\\n            \\\"activityId\\\": \\\"8f036878-c750-44d8-94c2-d0ffa48c5023\\\",\\r\\n            \\\"subActivityId\\\": \\\"f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"ecd293b9-988c-49f9-988b-2c0d898fd0bb\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d5d7ce70-22dd-4c8f-9299-1dc49ba137bf ARID=8f036878-c750-44d8-94c2-d0ffa48c5023 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e8598cfd-9337-4d84-8e85-f63664cea4a8 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ecd293b9-988c-49f9-988b-2c0d898fd0bb \\u003e DN.FE.ExecuteQuery/f3a261d8-5aa0-45c4-8e2e-c9b4dc64eef5)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092142908357480448","occurred_at":"2026-08-25T06:51:54Z","signature_id":"02210cdb0477006b4ad2f63d27bbb22b1fec432dfc421b3f4927ea8331e71bd5","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","signature_id":"ae0283cab4bf11ad3998038231a262c024b0853fa925e9504e4d72e900d671b8","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092131475158732800","occurred_at":"2026-08-25T06:06:28Z","signature_id":"8f69b74b7621ff0b0b8ce3121e6614ce9d3311a7090b53f7507107351ee1dd64","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092130874219827200","occurred_at":"2026-08-25T06:04:04Z","signature_id":"09aaf44a85636c53922c8c55c3f077fa7ba9e0be3e43723e44ff0ee719184329","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092122658786250752","occurred_at":"2026-08-25T05:31:26Z","signature_id":"0a451d1043de67f733809747c484d6352481472e46de0c7c6b4a4bb3a65e52d5","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092101471658053632","occurred_at":"2026-08-25T04:07:14Z","signature_id":"f34a1ababae19d28287c5f9ad6fadbb834adfab66885b9336868ab7e5fe3c4e0","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","signature_id":"ec3713fc3fd2cd0078c515157ab16cbcdeafc43ac2e38c3ebeac3729eec40f03","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092091165758197760","occurred_at":"2026-08-25T03:26:17Z","signature_id":"0656eacbaaee9c8627f179735f527bbfa1a360407e229ef55d81deb432be6cbc","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092089899510403072","occurred_at":"2026-08-25T03:21:15Z","signature_id":"b479792efcddc2bca8690264151fa877f3ce0d483876238bd57c9976805c7430","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092071627658366976","occurred_at":"2026-08-25T02:08:39Z","signature_id":"c9f1b4828c5cc783bc0db2584b06c52c8e5851d35045a6f094b2bdb76e6aa487","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092071623086575616","occurred_at":"2026-08-25T02:08:38Z","signature_id":"28297427d9d2bd8653dc649081b99f9573b907029f55909add770c57baa41099","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092066535932694528","occurred_at":"2026-08-25T01:48:25Z","signature_id":"002c1d2e05187089a12c62fceba5d319b6b2b88aa8f59eba28a6c84b20651f56","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092066407356305408","occurred_at":"2026-08-25T01:47:54Z","signature_id":"805bce7e0aab4147685c771f0375ca865818fb3186f1e427a03da21aa4561fc2","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092041100716937216","occurred_at":"2026-08-25T00:07:21Z","signature_id":"938c8e2babfc879ca08c00539016e48f5863191d883d5d8b5e82fc0bb437190d","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092034671041843200","occurred_at":"2026-08-24T23:41:48Z","signature_id":"93cd087eb0fc0ac2b0c79ba3bc6c6181f71cba9db1ac3eaefd4c1d8152fbb877","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092031971118026752","occurred_at":"2026-08-24T23:30:58Z","signature_id":"176c7f8e0e9a6fe4e5a12f9055659eede136ab94bf2bc621167d758c6dc1828c","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092030482601480192","occurred_at":"2026-08-24T23:25:09Z","signature_id":"a4d769910ea57a048ae6030e8851eaac93f508a9d62968944253d616d8fe88ed","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092025885698822144","occurred_at":"2026-08-24T23:06:53Z","signature_id":"9eb8d09c21becab11e4df2e2486fbadfcfe682a12484ff5957cc08b54e0cd902","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092020242174709760","occurred_at":"2026-08-24T22:44:28Z","signature_id":"a99ad7aa848b9f85d32a71fb35dfd49c3d8270fdc830d4478b6af42e4702a425","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092016236664721408","occurred_at":"2026-08-24T22:28:33Z","signature_id":"0d5221b00aeb7bfdca1da1581a9d7578e9d19b098899c13f87a46837f2394f22","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092014812023230464","occurred_at":"2026-08-24T22:22:53Z","signature_id":"c5fd89f1c41b331bfa0c3c5ff918b5ee94282a73960beb6c8680dfabd0d92aba","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092003732647579648","occurred_at":"2026-08-24T21:38:47Z","signature_id":"c3920cca5c3b22e9ea0a8df63d7fd82dff370a2c49fd2a56d2b8612e6611e8cd","pr_number":6088,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091996785936437248","occurred_at":"2026-08-24T21:11:15Z","signature_id":"c79de5565353120aa973caf9da2e4e0043d28fd25ac56a7cd204827ac9e612e5","pr_number":6128,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091995273302642688","occurred_at":"2026-08-24T21:05:15Z","signature_id":"b19e1030a263db434fc5938127062d3add16c8d3a56319f83b1663785d30bba9","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"98f2c85bfff864b394f0a6540743e0876e4af34e9f265ec3b983e45e8ef64a57","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"fa1482e0c10fe89a03896ac5f14bc5ff11cbc2b5c024fd993535afa6403d322b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091985911028912128","occurred_at":"2026-08-24T20:28:03Z","signature_id":"43910364f48f66523f1f9d08a417492faf101b0ab2cb444442ab1a553439c6b5","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091985362346840064","occurred_at":"2026-08-24T20:25:41Z","signature_id":"176cc4504c2f048683d4e9fd8346ddda26590ba46f5fcff44ce3756a85f5565e","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"913731872ab076d2435270c0fedcc39b2d88a5ff36047161ca4eabe2b648b67d","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091978207036182528","occurred_at":"2026-08-24T19:57:24Z","signature_id":"f663b43ed0edc7e9f34c5b7d6058cf02871c4b2cbf7c46200eb2800bb395c1fc","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977503907254272","occurred_at":"2026-08-24T19:54:38Z","signature_id":"0b9a16666408f3f71c77e5a20e559de4e428d06efaede29056c5fdec679a0549","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977495384428544","occurred_at":"2026-08-24T19:54:36Z","signature_id":"07fcf787e9667bb548a1ae4e2aa892f82683b85aa913615ccc3c55f328e44f13","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977485536202752","occurred_at":"2026-08-24T19:54:34Z","signature_id":"edf4f7b0d5436e65624e47252a3c9819a947e53aeb06b94097e54f965d358ffa","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091965378614792192","occurred_at":"2026-08-24T19:06:19Z","signature_id":"69234d1190be6bae030c9f0be83f507930b8aba09ce9643964f39fe12af78ce2","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963250454302720","occurred_at":"2026-08-24T18:58:00Z","signature_id":"1bd7b81142bf5c01b0963c412981b707f78cb8184adf320516339e4aa3a3ce9f","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963205222928384","occurred_at":"2026-08-24T18:57:49Z","signature_id":"9bfada2b49b3fc5245efb1ec3e602fd604d8132a904f08045a044c170e893680","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"c742492599a708bf77717b7e4360828e91602b79b9e3f94c759bffe66f0abf90","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091960563839012864","occurred_at":"2026-08-24T18:47:19Z","signature_id":"ec35c9667d01d0fca7a963152324f60489cc7cd6ddc292c3798abbfb8aa51092","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091956227226472448","occurred_at":"2026-08-24T18:30:02Z","signature_id":"335052cc293f1dcfc9068c799f022fd529e07859d0cea778ae29ff1ed7adcc44","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091950955841982464","occurred_at":"2026-08-24T18:09:09Z","signature_id":"c0cebe2544e94f85956b2aa4e6cb37d39972915968a1824b87954f1d8f2b0278","pr_number":6661,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091948175697907712","occurred_at":"2026-08-24T17:58:06Z","signature_id":"2cfd9e8dd04e05dbf25d7770f4e1349a8d7b20bfb77b1b160b57373b94af3940","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091942092610736128","occurred_at":"2026-08-24T17:33:55Z","signature_id":"41ffa53c8fc1675a8e8f11ea4fff86270a85dca8d99ed458236b7319ea0aa4d4","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","signature_id":"dec2f42fbc616357b928df0a2e19436696b2e782470cc0447de0e33d58d63917","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932257060130816","occurred_at":"2026-08-24T16:54:39Z","signature_id":"cff96711d2e8bb5b9f48911a18c760f7b843abef0ee0b1364775c2b0b5f58a49","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932359686361088","occurred_at":"2026-08-24T16:54:34Z","signature_id":"5fa5e4ca1b4d0f869a74c660939188cb0fcaacbc16b74da34e727232c29abbff","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"6ff77bfa149b82860b0429a7dfe5ab2b71a58982ddc24dc6fbf5996e7d1dca8d","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"1ae3c1ffd864572007c734910a8d27ab7b2bea356002ed663c85c3e98b525248","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"9865c7135fd03ceb69a76994157b6614c9b64d8eebb69b3b00180111a55f75a4","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091926769790619648","occurred_at":"2026-08-24T16:21:04Z","signature_id":"40f2371fc4b699bfbeed8a52087620105728edb5da70261d5e9bd7df99b13e6a","pr_number":6639,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"040407fa33e9112f6de934bf11f0cc510a5b5344ada4499539ca4bdf70de14a3","failure_pattern":"ERROR CODE: InternalServerError; detail message [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":46,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":46,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,46],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f12f40\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-f5xqwdq9mzqv\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b936c26d-3470-4f47-84bd-2b15b3591b1e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b936c26d-3470-4f47-84bd-2b15b3591b1e\",\n      \"name\": \"b936c26d-3470-4f47-84bd-2b15b3591b1e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:49:19.625710695Z\",\n      \"endTime\": \"2026-08-25T20:08:26.92099949Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f12f40\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-f5xqwdq9mzqv\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b936c26d-3470-4f47-84bd-2b15b3591b1e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b936c26d-3470-4f47-84bd-2b15b3591b1e\",\n      \"name\": \"b936c26d-3470-4f47-84bd-2b15b3591b1e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:49:19.625710695Z\",\n      \"endTime\": \"2026-08-25T20:08:26.92099949Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e28d80\u003e: \n    failed to create HCP cluster np-dg-minor-zpsckm: failed waiting for cluster=\"np-dg-minor-zpsckm\" in resourcegroup=\"rg-np-dg-minor-zpsckm-7j842f42696c\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c906de65-e3a2-4879-9236-7b176a8d1899\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c906de65-e3a2-4879-9236-7b176a8d1899\",\n      \"name\": \"c906de65-e3a2-4879-9236-7b176a8d1899\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:48:38.123542143Z\",\n      \"endTime\": \"2026-08-25T20:07:46.9158638Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e28d80\u003e: \n    failed to create HCP cluster np-dg-minor-zpsckm: failed waiting for cluster=\"np-dg-minor-zpsckm\" in resourcegroup=\"rg-np-dg-minor-zpsckm-7j842f42696c\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c906de65-e3a2-4879-9236-7b176a8d1899\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c906de65-e3a2-4879-9236-7b176a8d1899\",\n      \"name\": \"c906de65-e3a2-4879-9236-7b176a8d1899\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:48:38.123542143Z\",\n      \"endTime\": \"2026-08-25T20:07:46.9158638Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e8e5e0\u003e: \n    failed to create HCP cluster np-downgrade-x827rx: failed waiting for cluster=\"np-downgrade-x827rx\" in resourcegroup=\"rg-np-downgrade-x827rx-59zprw89nzrh\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/601f12a6-e78d-4e43-816f-0a2f916a5e32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/601f12a6-e78d-4e43-816f-0a2f916a5e32\",\n      \"name\": \"601f12a6-e78d-4e43-816f-0a2f916a5e32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:48:49.295862053Z\",\n      \"endTime\": \"2026-08-25T20:07:56.866737842Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e8e5e0\u003e: \n    failed to create HCP cluster np-downgrade-x827rx: failed waiting for cluster=\"np-downgrade-x827rx\" in resourcegroup=\"rg-np-downgrade-x827rx-59zprw89nzrh\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/601f12a6-e78d-4e43-816f-0a2f916a5e32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/601f12a6-e78d-4e43-816f-0a2f916a5e32\",\n      \"name\": \"601f12a6-e78d-4e43-816f-0a2f916a5e32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:48:49.295862053Z\",\n      \"endTime\": \"2026-08-25T20:07:56.866737842Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"01e5501399c4ff9330ca8905c22cca453551f730bf7866919a23a1964d735140","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"071c52a55532188ca4d85bb7a66cc6fe5057e406feb7fa0eb18ba541618e910d","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"0b9d16275b3d7ff10a538398088cfd3b34d1f81c4cb68201208e8e4940acdb66","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"0bed7020ecd96a5cf05f223e463371381d77da2411a879375e1ceda63de28e4d","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"0ddc6a1225a6aa5e4af4f5f0cac4a1b2429896b3b2b8ecd24544d0bdee9d655e","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"137c0167738c3818a360a99bda994cbeeaf885b6a21dfa26c7f7e5381c79eb6a","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"150626fb402aa61e05e6c6f7e9bc8843969fdad7b45425051818fe06847d9754","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"15594220caf851c71a98818822ec8afd99496a6032084cc875914f18e55fca35","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"1b0503c4496d672dc2cfe86aa011acfb4fd95d2eec277298e9e02bf59cf65ab3","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"1babbd681ea180bcbac323e120929964548b008afb246353cbf1fad90e2045d0","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"227699a26b6d86c6ff9a362b2fd1395de244815866341bf1ceee8b2242c39b99","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"23e0583c28d9032ccce1bd36665f177c2f1db717f6e1cfd08f9ea2a16c2c6de9","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"2b7bfba4308decd6036acc7bc3a1df135b34ae217fab985b49db26b737a0d3a3","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"2bae653a8fb17447152d646aa0c2b4963ab31a22269313b92437db057bf036df","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"2c154aeee3d38aa0e65baa996b6067656fdcb76e0501965057b6de64078cad6d","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"37f401123d689a1b77a09fb3105cb5815a2f706e5e9877cf69d6ae68bf44f228","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"42009cf5afc6cd4364288595dd3fe884cb5f131f6903a745c42078e5d885c516","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"50e102a0b8a908cb91efb330c142288a7922ca39527756e228d95800edd32777","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"53f56986e4035859cdad5301f526d2a5f8fccefb68779c8f991838db02bdf8d5","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"5789fa442d4f8a0f4909b27b633de87680963e17d09512bd15731d2bd37905ee","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"5c886e13528881fff7670bd0116e318e6fdf76957dfb81580b348ec9602154a5","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"68545017c2b3bd81d0621bcf01c19e0de3848f03893c798a6c2aef8169f05af6","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"6bdc706667de6fd9046a69b69bd20920f987c7f6feff220cb363a388a2ebc362","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"6fe96d1ea335b0c185768786273c3d8084095250845c72ff17619116c26601e4","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"702d137fa1b1cc6cdf5e5a67bee5dcd22b5b57ed6578939ac00222e0d40e5187","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"86d19c78d0739357bd3f161b74003da30c80a9d54ee056afba56d5e92cfbbc8a","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"889088819d9dad08a48983198d0e17c9a48b8ac3d79ea908cbdb9bb35c082bc6","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"9bc356d5e4cf33cadfb9cc3dddabf59890f14d30c9b21e128d830f5bcd9b3092","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"a2da6b6b096cccb301a6ba7a740e8c3e59f518fd173797735af9ce7fdb6f0758","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"a6c110259b6db6815f32629ea3be7e1b59c5bf8c1cc0025a60f627f7d9a52eb7","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"ade895469bda3f6ff370f0428733134c672c1f332d14dbed05c869078e270076","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"b54b257434a08295e33fb0d9a84522989a2312ce08305e6584ba381ae359da3b","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"b67939caaf0419b44e1276fe638732fc70a0ba57b9c847d29eed62ccca5bd266","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"b933bdf4882baaf62138a22d1375d2afc64afb311d072ff04469393bbe10f8d1","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"bf05905ae686b69af2afa5478616503e24344e881559bb70f69d4c9570b0f7a8","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"c278280dc667ba84567309478370f71e50ba651e2a8a2399ea8b30d7333b71fe","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"c73655be7d93d003560cdc90a2bab9e176e6bcf65d8c644387408b36f348155e","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"d550d5cb6f50d0f9ff8f38ae93cfebf0a70e58547865e6dbf73e9cbe1c055656","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"d693677dcc2436c081b021447374afa98eb7158ee0d3dec68e19d233b84b5470","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"dee87de74156ea23b8b3520f533ef9190f72b7babcd20bcb8e2cf2a94d10dcdb","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"dfa42aba5c4cc2cef7d699b8c65ba25382242bb42c7e266ef350ef67bc80df08","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"e0b54c003aebec5d92b194a6f80ee906c128eb77169d7d2a7cba65e9aeeeba75","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"e5bdbec92086140fc310730009e23b06059442f01f5776e5f1192f7ae25b3f0f","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"e7f3a8d6fb04405286572a2d49c4ffc07ae39f040f9b3c52e8290ec4d6f9c045","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"e92d23b1277ca5f619831904feaa802051ec36b3b4c1e38928537ad0c359a4e7","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"f70d11b5e9cc7db70c1bd07ccd81a8b490b5e1aa926781339d876d128c6ce56b","pr_number":6681,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":42,"runs_affected":16,"failed_runs":16,"run_impact_percent":2.5236593059936907,"anchor_occurrences":42,"after_last_push_count":13,"also_in":["INT","PROD","STG"],"trend_counts":[11,2,0,0,4,0,0,36],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":36,"prior_last_seen_at":"2026-08-22T00:10:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-tlrb8m with version 4.21.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001a7a5c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-tlrb8m: failed waiting for cluster=\"np-version-upgrade-cluster-tlrb8m\" in resourcegroup=\"rg-np-version-upgrade-tlrb8m-64c6wd9hqbps\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1769c72e-78e8-48ce-b214-b5a858f938c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1769c72e-78e8-48ce-b214-b5a858f938c5\",\n      \"name\": \"1769c72e-78e8-48ce-b214-b5a858f938c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:22:26.310363981Z\",\n      \"endTime\": \"2026-08-25T19:41:32.435006844Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-tlrb8m with version 4.21.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001a7a5c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-tlrb8m: failed waiting for cluster=\"np-version-upgrade-cluster-tlrb8m\" in resourcegroup=\"rg-np-version-upgrade-tlrb8m-64c6wd9hqbps\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1769c72e-78e8-48ce-b214-b5a858f938c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1769c72e-78e8-48ce-b214-b5a858f938c5\",\n      \"name\": \"1769c72e-78e8-48ce-b214-b5a858f938c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:22:26.310363981Z\",\n      \"endTime\": \"2026-08-25T19:41:32.435006844Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-ttf5nhws49r9/cluster-candidate-4-20-fx47tw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009a08e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-fx47tw: failed waiting for cluster=\"cluster-candidate-4-20-fx47tw\" in resourcegroup=\"rg-candidate-4-20-ttf5nhws49r9\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b05be7f-16c7-4109-9394-3dc217cc5c13\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b05be7f-16c7-4109-9394-3dc217cc5c13\",\n      \"name\": \"7b05be7f-16c7-4109-9394-3dc217cc5c13\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:23:17.318805591Z\",\n      \"endTime\": \"2026-08-25T19:42:23.583486998Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-ttf5nhws49r9/cluster-candidate-4-20-fx47tw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009a08e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-fx47tw: failed waiting for cluster=\"cluster-candidate-4-20-fx47tw\" in resourcegroup=\"rg-candidate-4-20-ttf5nhws49r9\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b05be7f-16c7-4109-9394-3dc217cc5c13\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7b05be7f-16c7-4109-9394-3dc217cc5c13\",\n      \"name\": \"7b05be7f-16c7-4109-9394-3dc217cc5c13\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:23:17.318805591Z\",\n      \"endTime\": \"2026-08-25T19:42:23.583486998Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f14340\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-p6gkdz5j5kdk\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cd1f359-f586-4859-b136-632a31d92bfd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cd1f359-f586-4859-b136-632a31d92bfd\",\n      \"name\": \"3cd1f359-f586-4859-b136-632a31d92bfd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:23:27.431435186Z\",\n      \"endTime\": \"2026-08-25T19:42:33.505289846Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f14340\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-p6gkdz5j5kdk\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cd1f359-f586-4859-b136-632a31d92bfd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cd1f359-f586-4859-b136-632a31d92bfd\",\n      \"name\": \"3cd1f359-f586-4859-b136-632a31d92bfd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:23:27.431435186Z\",\n      \"endTime\": \"2026-08-25T19:42:33.505289846Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"a2fca96d5c30f50d48abd3d91900db1407214ad4c57c888c781660c67cdbdfbe","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"1f94d94c8e70a8e7a8b56a5653939cf4935a72854bc9feda9124071da3fbfe56","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"2ce22b33b27841f4451ddbf18b57eaa0818348b0deaa93898d8394e7df7e56d0","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"30dad826455fa0938e7cec0e4f031d259f61105244ff995b4e802cf11d598f9a","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"c0dcf5c3ac2881b29fcaf6a19b5d8e99b79dd0936f8a80948a5bd717cba76449","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"521ed306a26b9f9dacf6c47c753c84342bde38e73eb4c5ef23908a8ca2f86d45","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"8a342e0125bf2802c9ea767d7ffb9709438a1f4e72f2ad64f7e052e12c236331","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"e51e9dfee4ba4c9daa2b79ed992ce895a67497a0787c0bbfe0188313ac862ed6","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"fdfa7061d24f7ad3a467bcd48be893b74bb83304ac5cba05b613cd11d788255d","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"236662ab2cf12e0855b7e39f0def5a11fc29e4cd526454501035df011b642ee5","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"c1d27f93ce1a22b612a32738d89512a0f0eca7cf6197d0e5f557147d4b9ce477","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6676/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261481813905408","occurred_at":"2026-08-25T14:43:04Z","signature_id":"12cc269491c81e464e81232257f84d9ecbcb60190f84d4997ab10025e4d39380","pr_number":6676,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"489782a342d805c0338dec6f638d02703430b0e8f541b6a5142fbe16ac3c7ae8","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"93381f18ce633ba4d082796c83975ebeab65d65a6f8d890e5eb3fce8495ce24b","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"3c8445447816fab0494fb15002cb32f86c4fc7cc3167f611c626f8d9af1ff88e","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"3f0f7b3d38506c573a5570dd4fc273d4974d9a5328b2080e03b587e59e9463aa","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"0d1f7426cf173fcfa700b32a523143bec8203aa837293f2ac55c31004cab129a","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"99dd3763374e539920c960007cd1b14c1c67fd84b0f8ee47d35b2abd7ec3c4a2","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"82e0621d077685d0022747285dcb794c36f142bd42f8f8a172a77dcd55aad3e9","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"a12a937f6ae89e3736e75bd6e94756e9cbd733f875fec3c20ccc3e4632e8284a","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"d36de0542872df4f6d63fdbe3389505290d5b0baebed82cdd90fedf806ec3e01","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"431ff7e575eec123143334467cdb2866bc6f04d93674992976852f19ccc99809","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"4aece071fa661e61fc57264421a76c9eac13620f5e30bdc56e63af5b1984f5a1","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"983b22f1f6c46c18080a9b4eef0fb721f28445b2ab45c04695540c6e1542253d","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"d7b472469e4b347e99e5eb839d0b4c4d6750f5bc580fe678d89b0781d5908848","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"09789f559d5df6a2a46fb75a6ae925fba0fbd303143b135e8e8bf18f560ae90c","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"17edb3dd260ea059c244167666ed36031cba43f5f67e78698ae6697d0e919add","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"2b601911adad56c98c50dd26e4c497713a18ded9cd009a058cdef7ea37a3ab49","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"46cface1afceb818229ce74688d353ed8c60a22a002f15862ba902e03ea766aa","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"d0e594c3105542ca3fc8d2a4997071027f9acfcf8cc74b3eb6724ac1ff51a864","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"87423fb812ad03870c503443bdcecefa13a470d2f72adb08656f725ea5a893fd","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"9d9e7c31a0eab79ec1990c199b21ca4f9e3b8abb4900261837125436f88781d9","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"b6676f8c8a5cec645c6a41c81541d0d21fdc763921f2f8f5e84133e0d2a0b697","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"3be09b3efaa1d6639c2674bcbca81f107fd2e424dd0133709cfc55b04fc0a27c","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"48ff412681e68aef8f22697d9b84a39dbb80151cba4ee2ed83b622757bccfcd8","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092188089500831744","occurred_at":"2026-08-25T09:29:48Z","signature_id":"772bc28982c38b2a6bed58697050c102f7ccd329ca804037b2e6053a2681cbd4","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090954745601593344","occurred_at":"2026-08-22T00:10:34Z","signature_id":"9fdd133af6242db076a4b863aa1d464f03e0be3d2425306bd6ebd2a2737c472d","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090954745601593344","occurred_at":"2026-08-22T00:10:34Z","signature_id":"ca8616230c8094e8215035c77732ff5477c5550d89406ec868bafc87db992ea3","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090954745601593344","occurred_at":"2026-08-22T00:10:34Z","signature_id":"d4f90a79a8a4ef80c12861acf6f718ca11e3feb268ed3a504ce6369876d11459","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090954745601593344","occurred_at":"2026-08-22T00:10:34Z","signature_id":"fc5eacd8f09f9d4725654010946fc92d203916fc49bf7d302d0ea51293543de8","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"71f14ae85372860033787bd468b4f0331d04c4854ae7cf27ac26f9ab2d4916bd","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"8b23ec894c227e8f12470a818eb300ee9ff167afc7d296ad5b545a1ec14f8a5e","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":38,"runs_affected":24,"failed_runs":24,"run_impact_percent":3.7854889589905363,"anchor_occurrences":38,"after_last_push_count":9,"also_in":["PROD","STG"],"trend_counts":[3,2,1,1,1,0,4,29],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":28,"prior_last_seen_at":"2026-08-22T00:10:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0007ab4d0\u003e: \n    GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/805512f7-5822-4d4a-b7f2-904c7a51afce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/805512f7-5822-4d4a-b7f2-904c7a51afce\",\n      \"name\": \"805512f7-5822-4d4a-b7f2-904c7a51afce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:32.391153149Z\",\n      \"endTime\": \"2026-08-25T19:39:32.410048712Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0007ab4d0\u003e: \n    GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/805512f7-5822-4d4a-b7f2-904c7a51afce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/805512f7-5822-4d4a-b7f2-904c7a51afce\",\n      \"name\": \"805512f7-5822-4d4a-b7f2-904c7a51afce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:32.391153149Z\",\n      \"endTime\": \"2026-08-25T19:39:32.410048712Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fae120\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pqxcnq8z9jpd\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd31e0f1-3a30-4284-a7ce-0a370221b212\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd31e0f1-3a30-4284-a7ce-0a370221b212\",\n      \"name\": \"bd31e0f1-3a30-4284-a7ce-0a370221b212\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:31.984108593Z\",\n      \"endTime\": \"2026-08-25T19:39:32.326305358Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fae120\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pqxcnq8z9jpd\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd31e0f1-3a30-4284-a7ce-0a370221b212\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd31e0f1-3a30-4284-a7ce-0a370221b212\",\n      \"name\": \"bd31e0f1-3a30-4284-a7ce-0a370221b212\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:31.984108593Z\",\n      \"endTime\": \"2026-08-25T19:39:32.326305358Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00019ed80\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lxqrtnj4sd4l\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6bb31d6b-0f83-4e8f-801b-d83aa356a651\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6bb31d6b-0f83-4e8f-801b-d83aa356a651\",\n      \"name\": \"6bb31d6b-0f83-4e8f-801b-d83aa356a651\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:28.693998542Z\",\n      \"endTime\": \"2026-08-25T19:39:32.413356519Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00019ed80\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lxqrtnj4sd4l\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6bb31d6b-0f83-4e8f-801b-d83aa356a651\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6bb31d6b-0f83-4e8f-801b-d83aa356a651\",\n      \"name\": \"6bb31d6b-0f83-4e8f-801b-d83aa356a651\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:28.693998542Z\",\n      \"endTime\": \"2026-08-25T19:39:32.413356519Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"1a10e9f0d0ebd327c61e8e899905b12b28ad26f43e7f89a4aa098b09ef35fc21","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"569936c5ee5c9e4cdd5de51f80b6ee2ec197ed73598a8d190cbd9e81f0f34f1b","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"bf78b2adb206b76ebe7b562c8f5d0dcff4cb6f69f3835690af34a77d874060a1","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"c9efb511f1cb5ecb77052682b34bef74a06a5d809c163ae9d53ba77fc8559f19","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"7e61cb838af28069775e4047abe596d948ba92c34b9bf7479496b56271f567b9","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"a5efaacf08081a9c1c8f925d50e651cf29e6bdbb45a3190694979b6ed844c618","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"8467082f94f1cddceb7436a951e2eb6e01b80042e0072d9c5e4e2e131f794e55","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"daeffcbe55429e419c5333569b8df49743acfca0ee5ac2e58304d2421599fcfb","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"383b14a099f2ce5791796cf71b4050d10ad8f6db01c076f62c792d982c8b659a","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269882870599680","occurred_at":"2026-08-25T15:16:27Z","signature_id":"320a0a462e18ca42da7b1b26ae6a21190926e34c2244492c5c4e0a1c81102a82","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"148bb476ee1b7952c4359b6ada2341d8424d4da89438cb857f8b6bff27286eb4","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","signature_id":"0238da5b4ee608676a62f88216ac5a7d91c16da91f1ec76842258c4d72bebf4e","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","signature_id":"19f2511c94e5b6ca21cc71bd63aabc9053c7c653b42b617ec879ca395e30580c","pr_number":6630,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"6cd3b4ec3a78c5076ca55dacbf68ec344e1698246fdf7d2fed60e6b4e6630276","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"31be970aed5c1350ca97847fcd67fc79e890462b95ed0fd2c70eb16c601cc4af","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"da7c4bc1633eca36853e5843a03bffeb0f874d7d6c7c76694adad89844da3845","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"3397e3c177e331b0b3c56e3aca57d39b4697fcc3c0632de87b50427da89cf9dd","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"301d0f6d3f61c8fc4bfdf4b36d3da0f6a5670b9ba6b385d3c18bb6644f1afcaf","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"e70a8923551947fd69aed4662e45c5ae1a351669ab5e6230b53b2fb9b06f69bf","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"29bc2f93a6094b4a5d5cf641b428320d4890242a1af692aabdd89032ec92bca7","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"66a0672a0448bcbf1d60a1d5cafd7b371b74508101d9fc34d0777b13e4cf76e2","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"1c7bb16e4b644044bda0b75b866f85ba0a96450750139ae786fdc9571148e984","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"fd5838277e9b2418a0afbc2057c3ce5fc4bd346774696e4589e2c05f45f73f1e","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"09f9bb566a2cac0401d2761a3e203049bfc5d3f036f5254838e3a2c316495d34","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"0fc0765dde4e73e193cda291461bd615b21c604ca712b5db69f701c1f66f193e","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"313c6b86047e9e386b672d793f4b878d127f3c65a84326d4cc2a6231581f8bc6","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"643ac15ebd4f500d7cdfd1c31ab51409301a12ee67319d5fe193f4ec82a3edb7","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"b2a39dd5fceed70e654ab8c04be8e55e67bd4e3e353048a2a56cb92b2721b5ad","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"f5dd8ef4e673ae5c7d050ce26d2eedefb62e36c6ff1f4f21e2c53c0420687784","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"e79e05105a6dd37d39aa319e6b82dc1c85c13b7322507d230499b9d98521e94c","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"7f0730f0d1c7e0eb7a035ce2e1a824a7fb99f16bf281917ec8e0ea592b939bdd","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"f45d8f4fc578e9bcbc89c15b0ea557b08d4323587837d33c561c269a0b539be9","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","signature_id":"f749c71c85c572aa2c57394be8a931e71c30eda717753e81fa6456d997c9c5fc","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090954745601593344","occurred_at":"2026-08-22T00:10:34Z","signature_id":"023f4e4e9644f027df72df49213653e5102b64065aaeeb7d55a65b696305b4e0","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090643626764800000","occurred_at":"2026-08-21T03:34:17Z","signature_id":"417af16cda9f0be1e205be17ce85652108e7367e103bf854b8e12c7538a97141","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090314560790597632","occurred_at":"2026-08-20T05:46:42Z","signature_id":"ec821af0caf46bb7fcb64607776a1ac9117bee65d538374ed623769ca4db28c1","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"72ebafea3e11c609d2f9a462699160dab0bb18b50d4ec6bd66dc45358da28bbd","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"d8848b082a36fe7aa93b5aadfb34fc3693bade5b697b59af38856412db575df6","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":37,"runs_affected":21,"failed_runs":21,"run_impact_percent":3.312302839116719,"anchor_occurrences":37,"after_last_push_count":117,"also_in":["INT","PROD","STG"],"trend_counts":[17,12,13,10,1,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":125,"prior_last_seen_at":"2026-08-22T00:13:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0026c6318\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-krw8vkxmtzpb\" to finish deleting: GET https://rp.j5592064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74596e16-42cf-45f7-b5d0-faa23e1127f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74596e16-42cf-45f7-b5d0-faa23e1127f5\",\n      \"name\": \"74596e16-42cf-45f7-b5d0-faa23e1127f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T10:56:49.253326502Z\",\n      \"endTime\": \"2026-08-25T11:20:52.069849414Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdi5csg0pn2q641bbc4hk7ubn3ksvmp still exists (deletion dispatched at 2026-08-25T10:56:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0026c6318\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-krw8vkxmtzpb\" to finish deleting: GET https://rp.j5592064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74596e16-42cf-45f7-b5d0-faa23e1127f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74596e16-42cf-45f7-b5d0-faa23e1127f5\",\n      \"name\": \"74596e16-42cf-45f7-b5d0-faa23e1127f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T10:56:49.253326502Z\",\n      \"endTime\": \"2026-08-25T11:20:52.069849414Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdi5csg0pn2q641bbc4hk7ubn3ksvmp still exists (deletion dispatched at 2026-08-25T10:56:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001bdc3d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-k276rwlx6nvv\" to finish deleting: GET https://rp.j4608896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95be590e-be2d-4405-b000-e47e729c722a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95be590e-be2d-4405-b000-e47e729c722a\",\n      \"name\": \"95be590e-be2d-4405-b000-e47e729c722a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:29:12.661843478Z\",\n      \"endTime\": \"2026-08-22T01:53:12.268348605Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sbairsosicqjiq127hgv6ukup2i9asv still exists (deletion dispatched at 2026-08-22T01:29:12Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001bdc3d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-k276rwlx6nvv\" to finish deleting: GET https://rp.j4608896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95be590e-be2d-4405-b000-e47e729c722a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95be590e-be2d-4405-b000-e47e729c722a\",\n      \"name\": \"95be590e-be2d-4405-b000-e47e729c722a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:29:12.661843478Z\",\n      \"endTime\": \"2026-08-22T01:53:12.268348605Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sbairsosicqjiq127hgv6ukup2i9asv still exists (deletion dispatched at 2026-08-22T01:29:12Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc002b983a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-8fhdctfdz8rw\" to finish deleting: GET https://rp.j0088320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1d2b6d51-1179-46e6-9785-09bffda18aba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1d2b6d51-1179-46e6-9785-09bffda18aba\",\n      \"name\": \"1d2b6d51-1179-46e6-9785-09bffda18aba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T22:37:19.756260961Z\",\n      \"endTime\": \"2026-08-21T23:01:22.169782395Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb7u2scpkh7308u4g26d4adn4v6f2fq still exists (deletion dispatched at 2026-08-21T22:37:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc002b983a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-8fhdctfdz8rw\" to finish deleting: GET https://rp.j0088320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1d2b6d51-1179-46e6-9785-09bffda18aba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1d2b6d51-1179-46e6-9785-09bffda18aba\",\n      \"name\": \"1d2b6d51-1179-46e6-9785-09bffda18aba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T22:37:19.756260961Z\",\n      \"endTime\": \"2026-08-21T23:01:22.169782395Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb7u2scpkh7308u4g26d4adn4v6f2fq still exists (deletion dispatched at 2026-08-21T22:37:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6669/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092189876085592064","occurred_at":"2026-08-25T09:52:51Z","signature_id":"4dcc3e0028de7e96928b0a1a88758b6efab2ee551db9968cdbece8675c668057","pr_number":6669,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6593/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090955578854608896","occurred_at":"2026-08-22T00:13:52Z","signature_id":"ea71a88634a16d0afbc9df293cf4726a2f2149536108ac3773b3c1495ab52cda","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"9cb8bbbd46d4ff7b8d8a9145481c0d6214ebf1fee2dd9871b79584eecc0ace23","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"ed56ae2522d4aa5d67496604431cbfa03d468518851d39e15f802a1713c02c70","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"c9b7084d1a6d0d9bf7767d41e2dffc0fe9a0c368f17fbd2ba52ca28c7fef9528","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849031038177280","occurred_at":"2026-08-21T17:10:29Z","signature_id":"28c8a564f7881af385b8b7f9b5484f7b699520a4ab948ec6980718c6018c0c72","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"78a303c77738b363a2fb051a390459c944a6c6506892466069ef310d887a73bf","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"e8da6919ba5cfbd1fb3e43dfe2e38e3fb9f1811073e2681ed20f13aaa672f01d","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"f7778fa6694474ab43ef140bac6b29ba7b40c11ac312df889617c83f251ff3e8","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"25f5d6f9a04ec10b0a212be6cba0955c5609c0d0248a7038cbecb0bdeb77becf","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"b6e0d020ce7138aead9b3214d404255ccec60fbcbcf4cf08b2dceeef66ee2cb5","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090612430223708160","occurred_at":"2026-08-21T01:30:11Z","signature_id":"6f65a26c62cfa1ddfc52a9edb6e78c7e84f29cf938bdb44ba828a131a29ffb1b","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"526dfc672e25122e7f8723ddcc2e06db50b45870112cb9c196a9ec2b74c0177e","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090497381098328064","occurred_at":"2026-08-20T17:53:09Z","signature_id":"669b518bb8bb2772c1633069821e688c83a58a278e8614898aa7d5d3a3893821","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"89c0ce8eaad0a52018f4811492cf780d403fc9c453f7b4120a99286a662876fa","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"8ec48adca2936da1cda6f46dd400d943903ebe2820f6d5d048144f838fb306bf","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"c0b52de30405f1ce85289aff26156b7972cd83d819d41fc68c0f6d416896168c","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"eaa451d983ea15579a77a41e048b7449200983b60173dcef401970a8df5f49b6","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"f186ce9f560dda14bcb06e1c713c1ef4f35902f5b2c4080bc81055330cea6fa1","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090473213871001600","occurred_at":"2026-08-20T16:17:01Z","signature_id":"7303da35a96fbcfc9b1e7b5f88ae3374d9ec8cfc0a32609b0d1207bd56d810ad","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090473213871001600","occurred_at":"2026-08-20T16:17:01Z","signature_id":"db4ffd88711ea8acaa394bcc753b2c312b02ba7b4aa456f3a34b6a04f2f7399e","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"6ffc943b94f0cf1e4d54823cd500d26e42b1dd3574092ad832c9467d03061c46","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"280c80547a16d77efdca2f79dc8afee598695f38c3aeaa0d818b0900296fa51a","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"3da52cfa371882cf7ceec3946de980d3513c1eb93b2b9ec3854cf2a632d3cd96","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"a96fcc5ef3389bbac06e463dd369b5888c6f3ff0d594f053e80261667944cc1c","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"4a72bbb712e2337ea1052f6b5b96c57b641dda1c52686aa38e3b945cc76b0c0a","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"7065b6a303a56f0747f08761d1761fb4fe32033ebd0de9f2a8c85471aeb8ea96","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"e882de52842635c93d4cef7d33a3a311986afa2764942b5596deabc845a3a945","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090111171486027776","occurred_at":"2026-08-19T16:18:30Z","signature_id":"f0ad9c22bf6fa48d6421f1b3522dd36f34546531d81a66e4fccb8efb04ee61cd","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087137016811520","occurred_at":"2026-08-19T14:42:58Z","signature_id":"ddc579d2118f83fc570d9a2f19f7609d1aec309883d03b2d39212f6f1f3543ce","pr_number":6553,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"337286aa4213d0c7cc64473c7e3e34c20f6acd5171260d91c52af41491d1f43b","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"a68839bcf6dfec9c97e979fa1937e27adc5a54ca6b682fd582d1dcc41b100773","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"4de85c8d82eaaea68218d8d636d979c1c2ec23176f984d427e09c48365fa8ec8","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"6bd6698f03122a7286ece459cd702827041a0d6998cdc483c51d2cc13c809912","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"07928273367f3cea76524c08cc39626ec0bb68cbd2c65cc52d78457055bc9d40","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"495783b014e34adc3be2999cc227819cc07e808e7fc136b141dce817a9754c46","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"c62c14d54dade7c8d4c702367d469896ff1ea7b078ac4a9da2f5f1b0df6c564b","pr_number":6602,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":36,"runs_affected":24,"failed_runs":24,"run_impact_percent":3.7854889589905363,"anchor_occurrences":36,"after_last_push_count":15,"also_in":["PROD","STG"],"trend_counts":[2,1,7,2,1,0,2,23],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":57,"prior_last_seen_at":"2026-08-22T00:13:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e46360\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qjpc76rq228c\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812b9f46-3042-442f-9212-72e785e2f21e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812b9f46-3042-442f-9212-72e785e2f21e\",\n      \"name\": \"812b9f46-3042-442f-9212-72e785e2f21e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:26:56.464409042Z\",\n      \"endTime\": \"2026-08-25T19:46:01.698082602Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e46360\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qjpc76rq228c\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812b9f46-3042-442f-9212-72e785e2f21e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812b9f46-3042-442f-9212-72e785e2f21e\",\n      \"name\": \"812b9f46-3042-442f-9212-72e785e2f21e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:26:56.464409042Z\",\n      \"endTime\": \"2026-08-25T19:46:01.698082602Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00019edc0\u003e: \n    failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-prhmmrf56mn9\" to finish creating: GET https://rp.j0715520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90e4482a-a8a6-47af-8a58-d4c088a6709f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90e4482a-a8a6-47af-8a58-d4c088a6709f\",\n      \"name\": \"90e4482a-a8a6-47af-8a58-d4c088a6709f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:35:42.014090967Z\",\n      \"endTime\": \"2026-08-25T14:54:47.709528737Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00019edc0\u003e: \n    failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-prhmmrf56mn9\" to finish creating: GET https://rp.j0715520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90e4482a-a8a6-47af-8a58-d4c088a6709f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90e4482a-a8a6-47af-8a58-d4c088a6709f\",\n      \"name\": \"90e4482a-a8a6-47af-8a58-d4c088a6709f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:35:42.014090967Z\",\n      \"endTime\": \"2026-08-25T14:54:47.709528737Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:739]: failed to create nodepool npdg-422-420\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004c9840\u003e: \n    failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-wpzzs6\" in resourcegroup=\"rg-np-dg-minor-wpzzs6-6jql8zwjxxlb\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73b2139d-e941-4022-8bf3-72abae50ef1a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73b2139d-e941-4022-8bf3-72abae50ef1a\",\n      \"name\": \"73b2139d-e941-4022-8bf3-72abae50ef1a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:35:50.588975119Z\",\n      \"endTime\": \"2026-08-25T14:54:53.875483949Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:739]: failed to create nodepool npdg-422-420\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004c9840\u003e: \n    failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-wpzzs6\" in resourcegroup=\"rg-np-dg-minor-wpzzs6-6jql8zwjxxlb\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73b2139d-e941-4022-8bf3-72abae50ef1a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73b2139d-e941-4022-8bf3-72abae50ef1a\",\n      \"name\": \"73b2139d-e941-4022-8bf3-72abae50ef1a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:35:50.588975119Z\",\n      \"endTime\": \"2026-08-25T14:54:53.875483949Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"0f5a2599c879fd0d600773df180f9eb5e17770f7cbb1109f8e5936709c16bd9c","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092244166900715520","occurred_at":"2026-08-25T13:34:16Z","signature_id":"52544ef6923df629cf38fa8f3d327d749d9b4fc26a68706642aec73ba7646ec7","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"44ad071145b6440bebd0a2dc3f06efae0013a7f9125c8d5d38fc3bea9f263863","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"2ee84b2de684d6bebbf318c8d5de7d1297cf8cffb2cae88ad176fe02897a3e46","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"3ee97bfc9952749cbcf036e0d8407c3873b852dade18372153b3dfc59818dac8","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"8d522cd2839df39682d21c0af249f6118f11366a2bf105536db9eb68cb5a9451","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092234666756018176","occurred_at":"2026-08-25T12:56:26Z","signature_id":"fba360027133779a17d7b5f7a04eb5d7f56e70b4761fe51394db3cfa7dded15e","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"0f33bde12ddadf9e7c9bfc135e1acbf84861720d826fda54f0ad89d0271fe465","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"1e5d4e64aee3bb0b78e1fb3308473036771fbfa7bf224bebfc3e1ddb47aca9c8","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"5ac2db27048619c0a83b9d739f995196c79b0762abf056bc200ed007d3557d7c","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"7ba4d18cd6273c16cfe80bffe517d25785f0d11a2228c4383f25ed32d2c438d6","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"96854033898edccbf2bb6b496287f357219f96280efefccded408cffb3a2f090","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","signature_id":"4806e756ed0578671bc1be64618a268b2aa1f613e22beef835379c8a977148f3","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","signature_id":"da59667e1510570b8a7920b7fcdf7406c1b2daaf77247d80291d5db3549304d3","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191916618682368","occurred_at":"2026-08-25T10:03:03Z","signature_id":"eb35914289e36ada82afb95d0842ac1cbf05cda80bee90852c8426b9626b9565","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"0ac0ed3b0a9732a181325ecdaa144e0e29914071fc6f1496b3b0c6f22509cdba","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"806a6cee782cfb1e97e02367c95a679ce5b8a00edfd0c58d876dd15273163cad","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"74b264555e08507bc1b8cd31ab8e6916b262467b913f5056245709dac63bf473","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"84726c82ed97016f12844a47373fafc8ccb52c61c42bd82f0b0f6c4092b047bd","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"9862cf1db12a585c6af07719f78d5cb393fa5c03f36fe7693b11f8ee1e06cab2","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"e3ff43e82a0f930ac10e5802807ad5e255f69a88161d6322c6013bce16cd034d","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"f8dc4e77835e39b0b6cc3a84dbc11fd7f2cbf76af6acd98523b7cdc62c8c4941","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"a9fc0ce0c34af3d53b841c0d03266056bd51cbfcda7aed993a0a0b308252dad0","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977503907254272","occurred_at":"2026-08-24T19:54:38Z","signature_id":"eb508bccb119e029161a871dee87adcf17a97ac2d0acf65b89eac6aede0f896a","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"29b0688020a02097d8409be35d56bc8e657d156f249e7e0111a076d22256e797","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6593/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090955578854608896","occurred_at":"2026-08-22T00:13:52Z","signature_id":"f723c1595727953323cb9fbfda183c1dc163da8a20f83d112c16bfb48a359e65","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090803404556210176","occurred_at":"2026-08-21T14:09:11Z","signature_id":"5d29df6edd24bf24825c111a643726aa35b3e47b76bb21cfd88adb9f8c198cf4","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090612430223708160","occurred_at":"2026-08-21T01:30:11Z","signature_id":"b92b8c9c37c7e37c897bea63a81f66153460a282d26942ce204cb4ed880d97f4","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090576352783634432","occurred_at":"2026-08-20T23:06:58Z","signature_id":"c1ea50b77f344e91f10a7709d63b68ac571c1fae52ae6c557bcbde6d0e3e6973","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"25c1614b8b68688875cf44e728f0eac99596a1b24838f886e170d041538bea9f","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090501897923334144","occurred_at":"2026-08-20T18:10:59Z","signature_id":"27c839aa327e50470cf56ca5936906afe6a900b7f124103d58137e60ab9f15a0","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090497381098328064","occurred_at":"2026-08-20T17:53:09Z","signature_id":"b0e35cbedbf30d1fba69356b51c668e445c0f866abb5ade28eeb3cbf4e7b6dac","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090484744067223552","occurred_at":"2026-08-20T17:02:48Z","signature_id":"0f90aceec820f2d550b3126a4ae2a3020109e8539c5f0cdecebafb3137413a11","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090483907651702784","occurred_at":"2026-08-20T16:59:32Z","signature_id":"859b652c2c687397467c692aff5a0fb14dd7e1b66f94076a5f0315ad529f937a","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090373215313465344","occurred_at":"2026-08-20T09:39:46Z","signature_id":"501dd06fef250a9b340590187f616f2aef38d9c4cc0b4b2d3fcf24ce1e2979fb","pr_number":6589,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089981796769861632","occurred_at":"2026-08-19T07:44:24Z","signature_id":"2a8c3f17b5d44463f41d98cc3eb1095019ba876a0c688336c2ff2416d9d79832","pr_number":6157,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":35,"runs_affected":35,"failed_runs":35,"run_impact_percent":5.520504731861199,"anchor_occurrences":35,"after_last_push_count":69,"trend_counts":[15,12,10,11,0,0,1,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":229,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":35}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T11:37:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T11:37:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T11:37:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T11:37:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.","alert fired 24 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T16:11:08Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:25:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-08-24T16:13:07Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-08-24T16:13:07Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:26:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 21:\n  State:       Resolved\n  Started:     2026-08-24T16:17:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 22:\n  State:       Resolved\n  Started:     2026-08-24T16:17:09Z\n  Ended:       2026-08-24T16:23:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet velero/node-agent has not finished or progressed for at least 15 minutes.\nFiring 23:\n  State:       Resolved\n  Started:     2026-08-24T16:17:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet velero/node-agent has not finished or progressed for at least 15 minutes.\nFiring 24:\n  State:       Resolved\n  Started:     2026-08-24T16:17:09Z\n  Ended:       2026-08-24T16:29:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","alert fired 24 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 17:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 18:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 19:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 20:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 21:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:53:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 22:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:53:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet velero/node-agent has not finished or progressed for at least 15 minutes.\nFiring 23:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:53:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet velero/node-agent has not finished or progressed for at least 15 minutes.\nFiring 24:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:53:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092198034401136640","occurred_at":"2026-08-25T10:30:42Z","signature_id":"f21b751e01203e4e27106dcb1932c067722032d928d9e8e29b55092b5008f5d9","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"367941eb104ee306ff1fe196c961bc8c4a584d69e0f31a0ee0696e981fbdf639","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"83f1bad8d518c7beb63bce55e051860d194f93a05a39bd45a36f26ea11c37de9","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"493cc43ac674b927d12a09d7344bb8455bf9bb59b8b34c2ade48063df28c770c","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894675127832576","occurred_at":"2026-08-21T20:11:52Z","signature_id":"4b5ff8ddc74fbca1ad06f8acb960b40e5ac5288bdc02906474049eec047acbac","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849031038177280","occurred_at":"2026-08-21T17:10:29Z","signature_id":"54718b687d279cfeef25d73a1bd15f4acbe2c392068026450ee15a69c107561e","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"2c84bb7c1b1025bf5aca49043832b9795d9a698197c24f25788bf11286858928","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"c9a68fbb944333ee25a753b5b0f81d4460bb338db83839755dd4687597170d5d","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090660118717796352","occurred_at":"2026-08-21T04:39:49Z","signature_id":"2618f72610682f43dd2cd2dba26bdc0127c8d0b0defed2ee2da18c76b6b9077d","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"edd75028aec82bdb3a00ffa4b080650ca91380325a410deca047f6956533bdff","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"85e7b46b9b7ff85d3a77c82be204f3e8ee605ac388a88750102a74b6fcfc8e11","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090643626764800000","occurred_at":"2026-08-21T03:34:17Z","signature_id":"c94a19cdb1a31ba9664c24415b96a5205d3ca0629c25480f0e83765cac75c579","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090612430223708160","occurred_at":"2026-08-21T01:30:11Z","signature_id":"bc8a93b0db0fe1ef3d350779090613683a43a036387d73cd70b9d66fe288f8ec","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090587456947621888","occurred_at":"2026-08-20T23:51:05Z","signature_id":"2991094f0824a50ebc0d788dd77119799c574f1dac8d1b4357e9ec4e67b7f828","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"d15020d9116463034a4be5a9d6d97582c79c8f179dc8049587e89517faf78588","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090497381098328064","occurred_at":"2026-08-20T17:53:09Z","signature_id":"9f4f056083aa68743b18cd9abb3eff1c8907f5504b1ce08ae3c69018666a2320","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"8b71cdf7aa2dd4cf64a1f93522e60dad9e67f08b57d138afe592dcb0edfaf800","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090473213871001600","occurred_at":"2026-08-20T16:17:01Z","signature_id":"1f2d648dc953e3539e09c6b34321427152afbeb6e7be3ea9fe5c07b1939bdd44","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090443849187987456","occurred_at":"2026-08-20T14:20:26Z","signature_id":"36b8f686529afbb4e5980b5c913953298961d9ffb6b9e06ed78dea224b732056","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090419688172425216","occurred_at":"2026-08-20T12:44:10Z","signature_id":"52ba71bccdd9820d2abd1b0c61181d75f220e8e4f06edd79e92c083588e7f3f2","pr_number":6617,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090373215313465344","occurred_at":"2026-08-20T09:39:46Z","signature_id":"5fe4bd31ea1c475a5ead0034adafd043b80ceb7a26a5465a58e37af7a35a146d","pr_number":6589,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"36f2100c2a1fe2d84fdef00ccf37bdcf7e79999878e0a2391eb45841bd6c6a3b","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"02e6579931a5425586fd1b99e79cb9403b00e6ad53c42c0f54c14a97d68e75c2","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090180258316685312","occurred_at":"2026-08-19T20:53:01Z","signature_id":"8e9b9cfa88833157f1361b4031ce44e1106e86384c5dbf97963b200765db60f3","pr_number":6614,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"b3b7a82dd8fc7c9453c13d5bac57f70bad1257901992ef7e3b00a768cf93ce29","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090126361980047360","occurred_at":"2026-08-19T17:18:51Z","signature_id":"5b7f46f1c74c9f7df302416b3ff8d6f9c3b15b2f31197a771207b8031862e1a6","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090119905717260288","occurred_at":"2026-08-19T16:53:12Z","signature_id":"4a381fdecdd0a4be7797b7b05cb5a2e67ba5334f09d2a876403d68397bddc7b4","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090111171486027776","occurred_at":"2026-08-19T16:18:30Z","signature_id":"536e9b5133376da7186a6910260a4ff7eb9410c6c312609565848cec9c00b9f9","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087196852752384","occurred_at":"2026-08-19T14:43:14Z","signature_id":"c8313ef1a30fd3399e30c4c34f4f78d6f3cd827c632099807a4c65aeed41251e","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087137016811520","occurred_at":"2026-08-19T14:42:58Z","signature_id":"b09c2b57753187375e9bb4bfbf655cf935649a379ae9145ab6a1173455b81add","pr_number":6553,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"6d5c6eda78682131a737b5a1673e8cff260b18e1476bf9fc6e7786614a2c8ea8","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090072497390096384","occurred_at":"2026-08-19T13:44:49Z","signature_id":"b3476258f174db773fca00d2a8a1840bd4d80a888f87e9c93bf4175b85362c15","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"f6b16d89151fe9aa188d216d1e54e2be6e1547185a4d587433d32485b63925b8","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090031884472946688","occurred_at":"2026-08-19T11:03:26Z","signature_id":"20f3b15ee34600ced7a0a232da78a03021e68bcf501decf6ca03614b467445e0","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"ee274618c50bb1e5934e1429251a60e64d1ea78f9e670a0c1bef6baea15b5b1b","pr_number":6602,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":35,"runs_affected":10,"failed_runs":10,"run_impact_percent":1.5772870662460567,"anchor_occurrences":35,"after_last_push_count":4,"also_in":["STG"],"trend_counts":[2,1,0,0,0,0,0,34],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":19,"prior_last_seen_at":"2026-08-19T18:39:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-4j5sc54kln8q/cluster-candidate-4-22-sf6h7k should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010e0360\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-sf6h7k: failed waiting for cluster=\"cluster-candidate-4-22-sf6h7k\" in resourcegroup=\"rg-candidate-4-22-4j5sc54kln8q\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de17841e-350a-4fc5-b9b8-7f89fd7f8e91\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de17841e-350a-4fc5-b9b8-7f89fd7f8e91\",\n      \"name\": \"de17841e-350a-4fc5-b9b8-7f89fd7f8e91\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:37.268675469Z\",\n      \"endTime\": \"2026-08-25T19:36:40.707957411Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, packageserver, community-operators-catalog, cluster-network-operator, cluster-storage-operator, certified-operators-catalog, cluster-policy-controller, openshift-route-controller-manager, dns-operator, catalog-operator, cluster-image-registry-operator, redhat-operators-catalog, cluster-version-operator, openshift-controller-manager, hosted-cluster-config-operator, csi-snapshot-controller-operator, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-4j5sc54kln8q/cluster-candidate-4-22-sf6h7k should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010e0360\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-sf6h7k: failed waiting for cluster=\"cluster-candidate-4-22-sf6h7k\" in resourcegroup=\"rg-candidate-4-22-4j5sc54kln8q\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de17841e-350a-4fc5-b9b8-7f89fd7f8e91\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de17841e-350a-4fc5-b9b8-7f89fd7f8e91\",\n      \"name\": \"de17841e-350a-4fc5-b9b8-7f89fd7f8e91\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:37.268675469Z\",\n      \"endTime\": \"2026-08-25T19:36:40.707957411Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, packageserver, community-operators-catalog, cluster-network-operator, cluster-storage-operator, certified-operators-catalog, cluster-policy-controller, openshift-route-controller-manager, dns-operator, catalog-operator, cluster-image-registry-operator, redhat-operators-catalog, cluster-version-operator, openshift-controller-manager, hosted-cluster-config-operator, csi-snapshot-controller-operator, cluster-node-tuning-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-grhvwwgnmxfw/cluster-candidate-4-20-s6k958 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00111c140\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-s6k958: failed waiting for cluster=\"cluster-candidate-4-20-s6k958\" in resourcegroup=\"rg-candidate-4-20-grhvwwgnmxfw\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2555a7c-e636-4588-b48a-b1225e2d8f06\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2555a7c-e636-4588-b48a-b1225e2d8f06\",\n      \"name\": \"d2555a7c-e636-4588-b48a-b1225e2d8f06\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:38.527859766Z\",\n      \"endTime\": \"2026-08-25T16:40:38.616570204Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ignition-server, ignition-server-proxy, certified-operators-catalog, catalog-operator, control-plane-pki-operator, kube-controller-manager, kube-scheduler, cluster-network-operator, cluster-node-tuning-operator, cluster-version-operator, dns-operator, cluster-image-registry-operator, konnectivity-agent, redhat-operators-catalog, machine-approver, openshift-apiserver, openshift-controller-manager, ingress-operator, community-operators-catalog, cluster-policy-controller, openshift-route-controller-manager, cluster-storage-operator, olm-operator, azure-cloud-controller-manager, packageserver, hosted-cluster-config-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, konnectivity-agent deployment has 2 unavailable replicas, kube-controller-manager deployment has 2 unavailable replicas, kube-scheduler deployment has 2 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-grhvwwgnmxfw/cluster-candidate-4-20-s6k958 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00111c140\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-s6k958: failed waiting for cluster=\"cluster-candidate-4-20-s6k958\" in resourcegroup=\"rg-candidate-4-20-grhvwwgnmxfw\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2555a7c-e636-4588-b48a-b1225e2d8f06\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2555a7c-e636-4588-b48a-b1225e2d8f06\",\n      \"name\": \"d2555a7c-e636-4588-b48a-b1225e2d8f06\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:38.527859766Z\",\n      \"endTime\": \"2026-08-25T16:40:38.616570204Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ignition-server, ignition-server-proxy, certified-operators-catalog, catalog-operator, control-plane-pki-operator, kube-controller-manager, kube-scheduler, cluster-network-operator, cluster-node-tuning-operator, cluster-version-operator, dns-operator, cluster-image-registry-operator, konnectivity-agent, redhat-operators-catalog, machine-approver, openshift-apiserver, openshift-controller-manager, ingress-operator, community-operators-catalog, cluster-policy-controller, openshift-route-controller-manager, cluster-storage-operator, olm-operator, azure-cloud-controller-manager, packageserver, hosted-cluster-config-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, konnectivity-agent deployment has 2 unavailable replicas, kube-controller-manager deployment has 2 unavailable replicas, kube-scheduler deployment has 2 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ee2ca0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-bb6s68gx7kh6\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9561d49c-5877-4d79-855f-4df08a7e3fae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9561d49c-5877-4d79-855f-4df08a7e3fae\",\n      \"name\": \"9561d49c-5877-4d79-855f-4df08a7e3fae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:40.989997281Z\",\n      \"endTime\": \"2026-08-25T16:40:48.583774176Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, dns-operator, olm-operator, cluster-version-operator, hosted-cluster-config-operator, cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, catalog-operator, cluster-policy-controller, packageserver, community-operators-catalog, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ee2ca0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-bb6s68gx7kh6\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9561d49c-5877-4d79-855f-4df08a7e3fae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9561d49c-5877-4d79-855f-4df08a7e3fae\",\n      \"name\": \"9561d49c-5877-4d79-855f-4df08a7e3fae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:40.989997281Z\",\n      \"endTime\": \"2026-08-25T16:40:48.583774176Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, dns-operator, olm-operator, cluster-version-operator, hosted-cluster-config-operator, cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, catalog-operator, cluster-policy-controller, packageserver, community-operators-catalog, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"f5fbd5480e218821391ad606bc369871f81ab79ca13513e6166733cd7e61e250","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"1f2d818f791aabf7ca966417dcc515df6d86223731a47e82cb83edd0e59ba4f7","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"2e500e0b56bd09e2e3065c06d8cad98af6fc49618dd7b23cbffb2625133fb624","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"6d79c60b75c7247e2641f3408daf5532b9c4fe3a7b5290b37b8be3c077c8fcb6","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"62e1bcec648cf6ca85b4778d6a217f22d6ca9de165a453fb96f3ba95214f5341","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"69a7645ef69f7c63a48191f6533d04ca0a326a56df092b3558b2f664fafc5a80","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"86c42132d5d5019c3e30fa7d9fcde1ed01362afa33a0740202359c037a9d6aba","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"45742a02b7e0c32d86b97863f4ea299dc6196292e7ab7de047ae62dec5023312","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"722d515e29fb9ec826abf197c41290ecc658816cb59a8a63c0cebddb0232cee7","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"90684796a4aec3250755dd0c0757e80b9dd9a4d843996fedaf91a968864f7cc1","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"af3ee147c3245ccf45bdc2deacd994348ca6f9c719f39a02ed738a5c4ec6e6d7","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"e814323bff83120ddf30b4df3d591da6897add0520ab77b9aece5773322a790f","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"25b082e6f3fccd0eeb0248caee3439349733372ff0b9b1c11e1a4ee3a27cb02e","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"25be28244289ac46a2d410d588e4df19d0867d5c662db6e3ba488825169d0ea1","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"d17d79e8ac084d555f983d2abb36810032fbe5d1d3056157e89dc068b55ec851","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"26682ebc65e6771f4deee3248fa461bb0de542a4ab0eebfc70762ad29bf4f7f8","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"32b3f6d0980058b52ca8a27936b8fa3d683b85efd75564d377af68c930a114db","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"504ae63c8f089cb82a68d41adbc6f1b843f3855c8b1b37f224907c01e60811f8","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"812cfe1df0bd2944c0ae39c8f4dc119f0c86e898e12e7120fca50c7be2fd1a6a","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"8366f1f3e151dc6dc562cd5621b250932345234159c653b20f33f1095b4d2763","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"1e1bd91e2da41c37dab2549990d0b6c3580daa21ab6ad95e1315e6a8a8df6173","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"429e22ec2d21dbf61e0c7b6393566a722dc292f14b64db5f2803331be42369e0","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"732bac21dc6f40ccf236a6b2bee56772eb8544e2c5358966c3a2a55ba78a23ba","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"84df8ea31194f1a34ce54d507db6d896e44775caf63f19fedd4bd03f18b43d1b","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"a8c7c70bd3d6dd8630d347b4d24584ab9920ea0b8c33dde6ba59740ac0218935","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"f070923f4a7c1696bb4766ac883978b74411c5db4f343ca2e398a5b72fdf0f61","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"283564a0bebb53ed7bfe47fcf31d4874c7bd4b8c4a658e0aea01d9dd41b0242d","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"7a249edbd9da943618f89813673eaaae45051d2a658fe6c77b5dd31a98bcbc7a","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"8a57d6dcc713db3bc6dcc797a3a204959181e3f6745d603dd5d1db0fbf1450de","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"9e89748c5a442a7a3524b941a931bf754405a837847407c813dd70e1c49eea38","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"b2468c4e9d126aa56394e8ef9b7bce10bb3f3907f961e58663665d48e197b818","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"dcaf8967a5afa5f20ee5c32afe78d8acf920baee1936ceb3553c7140fba63e19","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"e27b3eb99bc3107ba8d48dd22285100fd839e258993d8e1be4af2f04735bbfd0","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"e3bec1f2cb1a89b8221c4f2e37ae1979157dfc028f4587eae931e2b14a591671","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"abd84fcd86a93cb5f5771dc01297e0cdabb7c3c66cb8870add8499b52e97b83e","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":34,"runs_affected":34,"failed_runs":34,"run_impact_percent":5.3627760252365935,"anchor_occurrences":34,"after_last_push_count":7,"trend_counts":[0,0,0,6,0,0,28,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":34,"prior_last_seen_at":"2026-08-21T03:51:03Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":8},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":5},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":21}],"full_error_samples":["time=2026-08-24T13:26:56.540Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:26:58.762Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T13:27:00.485Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=abeb20b90f6fb4be4d4a69c08ed3bbe8b5be5f3247506ae425e325b85b345969 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8084480-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fabeb20b90f6fb4be4d4a69c08ed3bbe8b5be5f3247506ae425e325b85b345969\ntime=2026-08-24T13:35:32.482Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/abeb20b90f6fb4be4d4a69c08ed3bbe8b5be5f3247506ae425e325b85b345969/operationStatuses/08584140288659502671\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/cluster-kyv757fphd2fm\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j8084480-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n                    \\\\\\\"details\\\\\\\": [\\\\r\\\\n                      {\\\\r\\\\n                        \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                        \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-24652102-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-24652102-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n                        \\\\\\\"details\\\\\\\": [\\\\r\\\\n                          {\\\\r\\\\n                            \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                            \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n                            \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n                          }\\\\r\\\\n                        ]\\\\r\\\\n                      }\\\\r\\\\n                    ]\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-24T13:31:41.774Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:31:42.863Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-08-24T13:31:44.020Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=cf8b4dcb00b286eb41bc03071739c50d179926b7125e9927b43e768ae20b10a5 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5557248-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcf8b4dcb00b286eb41bc03071739c50d179926b7125e9927b43e768ae20b10a5\ntime=2026-08-24T13:39:15.842Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/cf8b4dcb00b286eb41bc03071739c50d179926b7125e9927b43e768ae20b10a5/operationStatuses/08584140285820210806\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/cluster-ce5kkcjcnxbu2\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5557248-mgmt-2/agentPools/infra1\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n                    \\\\\\\"details\\\\\\\": [\\\\r\\\\n                      {\\\\r\\\\n                        \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                        \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-20013346-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-20013346-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n                        \\\\\\\"details\\\\\\\": [\\\\r\\\\n                          {\\\\r\\\\n                            \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                            \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n                            \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n                          }\\\\r\\\\n                        ]\\\\r\\\\n                      }\\\\r\\\\n                    ]\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-24T13:26:28.921Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:26:30.957Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T13:26:32.602Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=9508f110ddf9b49e79a1c674ffa6223d05182dbd61375e2ba689c04c8113e848 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2132480-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9508f110ddf9b49e79a1c674ffa6223d05182dbd61375e2ba689c04c8113e848\ntime=2026-08-24T13:34:04.585Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/9508f110ddf9b49e79a1c674ffa6223d05182dbd61375e2ba689c04c8113e848/operationStatuses/08584140288938416315\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/cluster-d54rkmqgu5xlq\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j2132480-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n                    \\\\\\\"details\\\\\\\": [\\\\r\\\\n                      {\\\\r\\\\n                        \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                        \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-42034125-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-42034125-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n                        \\\\\\\"details\\\\\\\": [\\\\r\\\\n                          {\\\\r\\\\n                            \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                            \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n                            \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n                          }\\\\r\\\\n                        ]\\\\r\\\\n                      }\\\\r\\\\n                    ]\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091873075288084480","occurred_at":"2026-08-24T12:59:40Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872721045557248","occurred_at":"2026-08-24T12:58:16Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872693082132480","occurred_at":"2026-08-24T12:58:09Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872666397970432","occurred_at":"2026-08-24T12:58:03Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091868107806085120","occurred_at":"2026-08-24T12:39:55Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6585,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091867251824136192","occurred_at":"2026-08-24T12:36:32Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6585,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091856057667162112","occurred_at":"2026-08-24T11:52:03Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6585,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091852685220253696","occurred_at":"2026-08-24T11:34:18Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6424,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844440330080256","occurred_at":"2026-08-24T11:05:53Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844377495212032","occurred_at":"2026-08-24T11:05:38Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844284889174016","occurred_at":"2026-08-24T11:05:16Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091840235057975296","occurred_at":"2026-08-24T10:49:11Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091827867787726848","occurred_at":"2026-08-24T10:00:02Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091827277728845824","occurred_at":"2026-08-24T09:57:41Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6429,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091823405572558848","occurred_at":"2026-08-24T09:42:18Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091816375096971264","occurred_at":"2026-08-24T09:14:22Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6429,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091814034880860160","occurred_at":"2026-08-24T09:05:04Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6517,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091812979992432640","occurred_at":"2026-08-24T09:00:53Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091803125458407424","occurred_at":"2026-08-24T08:21:34Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091801419551084544","occurred_at":"2026-08-24T08:14:51Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091795137024757760","occurred_at":"2026-08-24T07:49:58Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091792609700745216","occurred_at":"2026-08-24T07:39:56Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6278,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6656/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091776267920609280","occurred_at":"2026-08-24T06:35:00Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6656,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091774664165560320","occurred_at":"2026-08-24T06:28:37Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091750724990930944","occurred_at":"2026-08-24T04:53:30Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091747201922371584","occurred_at":"2026-08-24T04:39:30Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091734160543584256","occurred_at":"2026-08-24T03:47:41Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091733541242015744","occurred_at":"2026-08-24T03:45:13Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6646,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090647847190925312","occurred_at":"2026-08-21T03:51:03Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6619,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090611472622161920","occurred_at":"2026-08-21T01:26:31Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090609877989724160","occurred_at":"2026-08-21T01:20:11Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090609408819073024","occurred_at":"2026-08-21T01:18:19Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090605238405828608","occurred_at":"2026-08-21T01:01:45Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090596628044976128","occurred_at":"2026-08-21T00:27:32Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6278,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e4f1a1ea46940043a3767809b952d9be40a221815d3f81ceb038491c52599831","failure_pattern":"alert [svc] KubeDaemonSetMisScheduled fired","search_query":"alert [svc] KubeDaemonSetMisScheduled fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetMisScheduled does not fire","occurrences":29,"runs_affected":29,"failed_runs":29,"run_impact_percent":4.574132492113565,"anchor_occurrences":29,"after_last_push_count":65,"trend_counts":[15,9,9,10,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":207,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetMisScheduled does not fire","occurrences":29}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:23:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 4:\n  State:       Resolved\n  Started:     2026-08-24T16:13:11Z\n  Ended:       2026-08-24T16:23:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"7caa0010541a7c4ff937ecc7f1700635aa06e8c5ef418d2f4d2c09799f427a78","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"d85d02d9cb6d2c75be2eab26be90eebd98525d6443b746d221016f89fc701fb5","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"1f5d187ce251780f799514cf1123320091fcff9c835d9bfa0f2e34e82c80dc24","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894675127832576","occurred_at":"2026-08-21T20:11:52Z","signature_id":"fd26538be5506281b89a0f2bdc1f564962744e634a6f244b5fb748dff210306f","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849031038177280","occurred_at":"2026-08-21T17:10:29Z","signature_id":"3d7a57b08e70dfcb08eef19bdaef6376a8198c105bd85ba8dfaa49e85fcd4b23","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"823f5b2aa8486a7acbe1e124e698dd28e6761a0aee8a4da1641d137d9f965857","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"775472cbd3ba6274119d9bc4ff2a717c920fd383df136f13e36acce94c0bdfb6","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090660118717796352","occurred_at":"2026-08-21T04:39:49Z","signature_id":"08ee6d6fe4f72157b7e60bdf3099792d877119ac9d5be1390164d8d37baad484","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"67a7460dd8f9b8fd2618e9491341d8f7e95c65de245a49249b7bdd6d050e5828","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"17225edf7352999500f539996fb1e01462e7b62a705f105c43a815c2c5405f67","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090612430223708160","occurred_at":"2026-08-21T01:30:11Z","signature_id":"e27081bfeef47cc4451336118ca5c4a1a5e79e92b0462a62edd92bb442b7c0e3","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090587456947621888","occurred_at":"2026-08-20T23:51:05Z","signature_id":"85ae2a4c300dcd3c7cd534d580abccbc52517c8a99dcb6c729d5062e671c8397","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"9fe5db5cb089dd418c555924d53d7186d613f25ae9f6c4b559ff7d0fec98563e","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090497381098328064","occurred_at":"2026-08-20T17:53:09Z","signature_id":"c6814be68413071dff1b4160fa6464cab99366cb0d562af5e197e45927105455","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"153158a58298050b340d840baeefe1849b5a5f210965465e880847adee541906","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090473213871001600","occurred_at":"2026-08-20T16:17:01Z","signature_id":"70c2460bf40893e6a3319184633a62cd5392e1fa1da4d377e367e9688a26623d","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090443849187987456","occurred_at":"2026-08-20T14:20:26Z","signature_id":"5fffed194c8b221512b1de9118320c058d429ab88d6a98085b57636306ebda90","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090373215313465344","occurred_at":"2026-08-20T09:39:46Z","signature_id":"a59681b8fb4c8d535dfc9ebe1ce2f0bffe47d780f475ed48cd4e44fd1eaf7ec3","pr_number":6589,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"955bfb75d761e55c3947d67a37a34c6f865e93d558669c6e553c2c55fff53b0a","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"12c751faedf8b07d89a7b665d957a08a566082aedba6ef103a3ae86e8b16bbe1","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"61bb80579bb3f03389e124f657ef9a4580750698aaec8f6d4a5cb42fd9c5a58e","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090126361980047360","occurred_at":"2026-08-19T17:18:51Z","signature_id":"6d0c2cf5e46ab65efd59eafc90530d6afea5da68b455b1c9a1ed74acdea6ab01","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090119905717260288","occurred_at":"2026-08-19T16:53:12Z","signature_id":"f33fa63e75243f8f6c0662e8c432ef5ef1d02d6f6d8078b95a60a5b86a1e252a","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090111171486027776","occurred_at":"2026-08-19T16:18:30Z","signature_id":"ec22af5eee9a5decc3de2f2d105970ef83ec4375005a081c6224c7b8c664f906","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087196852752384","occurred_at":"2026-08-19T14:43:14Z","signature_id":"ab975ddc2ec3e21b23c47ba3911f432624317d5e2ab06dabdb76af0d67467eaf","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087137016811520","occurred_at":"2026-08-19T14:42:58Z","signature_id":"f4dc165345eaa96f69c587c27e604b2d16e6b78b53c717ec949a8df90b91519f","pr_number":6553,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"84e0dbe296baf5af2dc285e38ecf1486e8c1d78306f03ff21441b3348759b746","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090072497390096384","occurred_at":"2026-08-19T13:44:49Z","signature_id":"6ebffb8dbc25915240a6cfbfe96d94fbca4a5dc38b4159455c3c48ec63042680","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"fe495db1d221c4a8c124437bf73ed3f249a488629d3f37aad664611de5f9690b","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d6c6b690ca60846dd98e464313d6a9672b9abd8744604edd915627eb68203f9d","failure_pattern":"alert [svc] KubeNodeNotReady fired","search_query":"alert [svc] KubeNodeNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeNotReady does not fire","occurrences":27,"runs_affected":27,"failed_runs":27,"run_impact_percent":4.2586750788643535,"anchor_occurrences":27,"after_last_push_count":52,"trend_counts":[12,10,8,8,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":183,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeNotReady does not fire","occurrences":27}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T11:52:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"prometheus\", node=\"aks-system-30244119-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-system-30244119-vmss000000 has been unready for more than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T11:52:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.222:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1136640/providers/microsoft.monitor/accounts/services-j1136640\", namespace=\"prometheus\", node=\"aks-system-30244119-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wwh5l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-system-30244119-vmss000000 has been unready for more than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T23:03:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"prometheus\", node=\"aks-userswft1-18345013-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-userswft1-18345013-vmss000002 has been unready for more than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T23:04:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"prometheus\", node=\"aks-userswft1-18345013-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-userswft1-18345013-vmss000002 has been unready for more than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T23:03:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"prometheus\", node=\"aks-userswft1-16379006-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-userswft1-16379006-vmss000000 has been unready for more than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T23:03:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"prometheus\", node=\"aks-userswft1-16379006-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-userswft1-16379006-vmss000000 has been unready for more than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092198034401136640","occurred_at":"2026-08-25T10:30:42Z","signature_id":"7ebd8243470fe8b709d9e6b75f707a2eff5fb9d0f87bf99b81e7e336da9429c9","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"2d49350805ce859d57a8cd273a371aaa89e52a74ab0ec67b49ebbe6af5b52d3c","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"1dd5d3d158481f1b778faeeafc16d713505247bf52c9b2150dab32d0157936a2","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894675127832576","occurred_at":"2026-08-21T20:11:52Z","signature_id":"f426881115951519f9389af0904a3bed97befe021b18b71006baa4ac30465558","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"0d42f8d6d6935dd62ed8dbeb9c7e3e36edece6c01ae7f4ad36b523de5c16f59f","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"391e72b3f41ca31b44cc2156ee2d00504773a97d0b9001121531eb40acc62c51","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"50ed525a4e9c2c924b6e4c4dbd838ce38633b6d5809af050d67f8522a0857269","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"774245db4660623d4f87e4a9aab111e2ef5d0f1bbb16244e0f417dcd42b5cdc9","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090643626764800000","occurred_at":"2026-08-21T03:34:17Z","signature_id":"b1383ba37c59ad31a6376fa07ce2d216fd51357d35e203592cce52c4214a960d","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090587456947621888","occurred_at":"2026-08-20T23:51:05Z","signature_id":"03b8eeb862b3000376dbbf65c43e29d1d044f9cf45378cd0848c6ab5cae80fd7","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"16b785165c7398914a8e20defdd01e667418ba06bfc083feec0c48b05787871c","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"161ba18c834f66ee1dc848538f0020fb2526c47e9571c94c76bbd2cd532c5ac3","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090473213871001600","occurred_at":"2026-08-20T16:17:01Z","signature_id":"1f17d318d94c5fe2c71f0d60b8bf5c0656f1cbb6ed461985c8328a2136615e75","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090443849187987456","occurred_at":"2026-08-20T14:20:26Z","signature_id":"53acdaa3429f7031060b3d1d59848e2a333fb88cabb6b0058770adb211218484","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090419688172425216","occurred_at":"2026-08-20T12:44:10Z","signature_id":"fb0987f312f9186fe0c7047695760c4065c9de8f5c4a5a7d016ab884637adb85","pr_number":6617,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"471d7ef1f6787056c1e53c9f3e004ce11ca87bc63edfd5c891057ce365dfff2d","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"a82ec01c8cccff1f4eccc628f23282745da5fc11afcb1c7a3043656d526e3af7","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090180258316685312","occurred_at":"2026-08-19T20:53:01Z","signature_id":"78c9f7739387864e1541324ffece4d4ea39fa817ba903bfa58d7a29d048cf9f4","pr_number":6614,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"0b67643102a1580039188783547fb06811c61148f505d0ffead658e5b91844c1","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090126361980047360","occurred_at":"2026-08-19T17:18:51Z","signature_id":"830394846f5cac4afc09f57187503aef9e826ac2ac99258336feb5d1cb36bcb0","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090119905717260288","occurred_at":"2026-08-19T16:53:12Z","signature_id":"7ed39c4abba50d92c5c7504ba592834e0b1647a9a852214043e3fbe6989d14a2","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090111171486027776","occurred_at":"2026-08-19T16:18:30Z","signature_id":"3d5d2a5a3cc187abd2d61709b5ca5f2533906e15dc434ba2181c758948bd12bd","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"88965bd8d63b9b169b13dcd3437f55796c215100cbd854f4d3437c1cd42a0e11","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090072497390096384","occurred_at":"2026-08-19T13:44:49Z","signature_id":"8e3321d42d8aacd9fbb824acefdb20e6a55ee2552b45adeb84026dc9aaf03cf6","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"6ccf240cb2a00ffe7d808440e9b97f42f94c7c8b737432c23072b646299858c8","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090031884472946688","occurred_at":"2026-08-19T11:03:26Z","signature_id":"53dbfbe54fc3f1c917661518de4e9bc09c6d0a2092c07e440e67b2fe23bc8da1","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"9740659f3211c4e67e9eddf1451df47f3260c679950a37e9f027cb815969d67c","pr_number":6602,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":25,"runs_affected":25,"failed_runs":25,"run_impact_percent":3.943217665615142,"anchor_occurrences":25,"after_last_push_count":48,"trend_counts":[16,6,8,5,3,0,1,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":192,"prior_last_seen_at":"2026-08-22T08:48:46Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":25}],"full_error_samples":["alert fired 14 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T17:23:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.142:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn70pm4stdfpd62ob77dhptdq67h82-k7b0n1g7e9t6i9r\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-d2656\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sdn70pm4stdfpd62ob77dhptdq67h82-k7b0n1g7e9t6i9r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T17:23:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.142:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn70pm4stdfpd62ob77dhptdq67h82-k7b0n1g7e9t6i9r\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-d2656\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sdn70pm4stdfpd62ob77dhptdq67h82-k7b0n1g7e9t6i9r/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T17:29:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T17:29:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T17:29:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T17:29:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T17:29:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T17:29:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T17:30:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T17:30:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T17:30:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T17:30:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sdn6lk5i59n2vdugq91i3qhl1hovg7u-basic-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T17:30:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sdn6lgco9mj0rdvfe15crftvs8408gm-i3f3a1k1t0i4d3n/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T17:30:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0912128-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0912128/providers/microsoft.monitor/accounts/hcps-j0912128\", namespace=\"ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-jcll8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sdn6vvfklofaj57683jl810auik73me-u1n3j6u1c5a9g2d/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T08:31:29Z\n  Ended:       2026-08-25T08:38:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6292864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6292864/providers/microsoft.monitor/accounts/hcps-j6292864\", namespace=\"ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-rzqj4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T18:15:21Z\n  Ended:       2026-08-24T18:36:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/capi-provider has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"e028319f9187bbe822ec5100ba5bf10f82963cc9e819d8d10a3fec4e07cba99c","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146717536292864","occurred_at":"2026-08-25T07:07:02Z","signature_id":"8c886dd26eb0284df24c3f192d77269e9116e884fbce7b0f37949a6fdc5f288d","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"21c20c66c7c9aa8660ddbc1f0e9827cf7f85ba5f2e371e49fb9660d2202ea0a2","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091085161763442688","occurred_at":"2026-08-22T08:48:46Z","signature_id":"ea45d80556c6df1b7dcdc93568199771da13a2e986e58392825c14f49b9dbf7a","pr_number":6640,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"e455d9b7b038a17e57cc035ca9c9c5f5a28eaf76b1c22688c29b969f7b31ee75","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090960143322714112","occurred_at":"2026-08-22T00:32:00Z","signature_id":"704ddf6adff4d039aa03923c06032bc68e7d6d5f5f219a74df97dc060fd8fc0b","pr_number":6639,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"351d71055a17078b68a005f194223b4a24156deb912d46427e913acd0aeb8777","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"2e25fc232dcb33241fcd0cb57973dad8d90728d5fe6dcfb8afc166949d3b12a7","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"0c7f8c5ea85d16164325cbe2e383117ecacd076d46ac3359cbd59f601532eb44","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"72701a355bc04323206fcd1c0bd2b36b578f4d899899554804fcd06228542320","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"b30701a5cf5cf655007b0c0e5396a33379506a4b879c38d39601b83fa7527486","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"f5e5986bcef7a7e209bf0fed58426b925ed47dfd3408962879b99b77392036c5","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"03d4ae8262f1770e01a1d35260bf8e34f79b37a5aaac55583e71924d479302f8","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090497381098328064","occurred_at":"2026-08-20T17:53:09Z","signature_id":"5db4b0d2292280947c2278091bb4c40b2d5b8ca5b79b2d7c07f2c2e0859a0850","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"677320e48c2626b64b19de7b052b3fb050b9e5acc1ec61005b4e06f51178e853","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090465920358027264","occurred_at":"2026-08-20T15:48:08Z","signature_id":"be8f5ddd90fd2dc0cabb580d1ffa28061209566ae2296abdc713b4d8b56c7b97","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"3754690aedcadcebdaafefd53de15e8476013f726ec2d2b70ec0ae536dea3439","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"fd0ea1182bc283fb8ca2a0c6dd6b26ae5bd1e3dad84a163a320485814a6ea5c0","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"ee648d4d3cb0ce098a24d4ad4a68319d02795b14bc529474fdb13e67830298d2","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090206132864815104","occurred_at":"2026-08-19T22:35:50Z","signature_id":"e4f5c665fa9bc4222d3c1be5d216af10b1bf7853150a7490188bfc4942188881","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"c63de78f05317712503aa77f111d1143e526f4ee67eb10bdd44087621b074be2","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090133175878553600","occurred_at":"2026-08-19T17:45:52Z","signature_id":"7a0e5fe89e8c70bbb67c4e405e89758b4a8d2fdfa426a3e6b132bc93a3b5a5ec","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090126361980047360","occurred_at":"2026-08-19T17:18:51Z","signature_id":"f28a6eded26a41ba3b81af27f2c8ab7a2fb450ba34046088f326e7c899b05e6c","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"48bd353f139412a8282a326a12d2c8de87fc5c59236878c955792c79218a1005","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090072497390096384","occurred_at":"2026-08-19T13:44:49Z","signature_id":"48394d74a095ad1f33da1153c255bd41f1fb141941e6fd8809cbfbe09968b204","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":22,"runs_affected":22,"failed_runs":22,"run_impact_percent":3.470031545741325,"anchor_occurrences":22,"after_last_push_count":29,"trend_counts":[10,5,6,5,3,0,2,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":125,"prior_last_seen_at":"2026-08-22T08:48:46Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":22}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T08:17:26Z\n  Ended:       2026-08-25T08:33:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6292864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6292864/providers/microsoft.monitor/accounts/hcps-j6292864\", namespace=\"ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-rzqj4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-25T08:17:26Z\n  Ended:       2026-08-25T08:36:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6292864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6292864/providers/microsoft.monitor/accounts/hcps-j6292864\", namespace=\"ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-rzqj4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T18:01:25Z\n  Ended:       2026-08-24T18:34:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T18:01:25Z\n  Ended:       2026-08-24T18:34:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T17:55:40Z\n  Ended:       2026-08-24T18:14:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0629888-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.209:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0629888/providers/microsoft.monitor/accounts/hcps-j0629888\", namespace=\"ocm-arohcpci01-2sd354mcvucng8mm6igmcq4k43caf9jb-s0d4l4d8t6j0p0t\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-tktc6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sd354mcvucng8mm6igmcq4k43caf9jb-s0d4l4d8t6j0p0t/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T17:55:40Z\n  Ended:       2026-08-24T18:15:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0629888-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.209:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0629888/providers/microsoft.monitor/accounts/hcps-j0629888\", namespace=\"ocm-arohcpci01-2sd354mcvucng8mm6igmcq4k43caf9jb-s0d4l4d8t6j0p0t\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-tktc6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sd354mcvucng8mm6igmcq4k43caf9jb-s0d4l4d8t6j0p0t/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146717536292864","occurred_at":"2026-08-25T07:07:02Z","signature_id":"1d97f68d37328cfbf8faf6e5077141064b5bf086d525d3989deb1486a0606173","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"97ee0c31f908f1dbb20c9b6757ed42d4855cc07c6e4c69aed451babbcc2b480c","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"99e8b0c7fce35f618653d3b9144ebffd6644e1f77bf1ee7a7e41e85ca3b0c8eb","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091085161763442688","occurred_at":"2026-08-22T08:48:46Z","signature_id":"23d14823c199d5901c0285f35564ec7461e2aa45ffdc1a27b21665c828219f0b","pr_number":6640,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"22183042c9ed30b84dbf6d39061ab4bc9df56e90ac5267a78267ac4493dba186","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090960143322714112","occurred_at":"2026-08-22T00:32:00Z","signature_id":"7beacbaf5f0e4bfbba1a2f2ad49cee0d638516eb819a023195eabd3599d5ec0e","pr_number":6639,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"46d4ece5f5d566a68e9c280231b88623d89a58a8337d07f4f4a7df19acc9fd13","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"99fe7c2c7c806ddd200c62659c7ae464504909d81a4495a62d531c5eefa98193","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"80ee9143b08193bf0a21a4ec6c3243441efc3dbf08661b31e12362f00673d7e1","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"2de1f116a35eddb03dc2ba7d4574fd71bc2baa693fc063c5ce35ef13b561da68","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"04de974286e7d1201173912d76047280a42dd798689b1612cc47a67ded0d727b","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"6ae1089324989f77aeb5a95ad4a7163481d5b47367c03569c71e81b3ac2beea6","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"4ef8d0aefb0cc453e9e9e28247d6a5123fcebf0ff7b94c102214341e80194200","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090497381098328064","occurred_at":"2026-08-20T17:53:09Z","signature_id":"6d7eaaa13690b75238e05b7d352b6a42cd8b1a10a0bdd33fded04570e22f0078","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090465920358027264","occurred_at":"2026-08-20T15:48:08Z","signature_id":"fc4a65ca072851cf04eb2c3585332a115cfca96e64ca1e278a1ec250c0a7be60","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090405439429677056","occurred_at":"2026-08-20T11:47:38Z","signature_id":"b4e4fcfee50aa357f28aa6f512f29bb34e2833c6f3129d1415d2343a83b72874","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"549491e61691d2966c0af2b0191ce2097e27428f0a5ae908e536be79c7c164e7","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090191599463043072","occurred_at":"2026-08-19T21:38:05Z","signature_id":"cf00ab67daf9fe871c7e62c5a966fc1052eca72c90f665ad5a93628612da6e0a","pr_number":6608,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"361977dd6db3385463de7b49ec92e3b08b56667c2651facab94103ddbe36fe62","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090133175878553600","occurred_at":"2026-08-19T17:45:52Z","signature_id":"2b8c33cd66ab1717f619ed8958087c1440b2c477957c77056db36eba09ec220b","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085264507539456","occurred_at":"2026-08-19T14:35:33Z","signature_id":"ffa6c82dc196170eafb22839b837c7e113ebd218f7e6741760d68be504b62d8d","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"4ff4aa5d104b7c3ca3e7f62fe40712f5a28a0dd3283492fb02c08dfec1c5ece5","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"519fe0fe7478bebfb7ef939c84f76fa0281886d85cb376148838a1878147b618","failure_pattern":"alert [svc] KubeNodeUnreachable fired","search_query":"alert [svc] KubeNodeUnreachable fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeUnreachable does not fire","occurrences":21,"runs_affected":21,"failed_runs":21,"run_impact_percent":3.312302839116719,"anchor_occurrences":21,"after_last_push_count":53,"trend_counts":[7,4,7,9,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":172,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeUnreachable does not fire","occurrences":21}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T16:12:59Z\n  Ended:       2026-08-24T16:22:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"prometheus\", node=\"aks-userswft2-20578738-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft2-20578738-vmss000002 is unreachable and some workloads may be rescheduled.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T16:12:59Z\n  Ended:       2026-08-24T16:23:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"prometheus\", node=\"aks-userswft2-20578738-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft2-20578738-vmss000002 is unreachable and some workloads may be rescheduled.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"prometheus\", node=\"aks-userswft1-18345013-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft1-18345013-vmss000002 is unreachable and some workloads may be rescheduled.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"prometheus\", node=\"aks-userswft1-18345013-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft1-18345013-vmss000002 is unreachable and some workloads may be rescheduled.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"prometheus\", node=\"aks-userswft1-16379006-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft1-16379006-vmss000000 is unreachable and some workloads may be rescheduled.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"prometheus\", node=\"aks-userswft1-16379006-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft1-16379006-vmss000000 is unreachable and some workloads may be rescheduled."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"8ea4612bb5240c0f449b27c94dc170f61dc652ef24bc74880e6489a1d15ea2be","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"962e0ef010bf48301058ca962542e7ac5e9edc6b0a882c5a67b669772b738665","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"2521e3c087abfbb2e0d499e90735681715a2dee5f344667b798d78dfae706547","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894675127832576","occurred_at":"2026-08-21T20:11:52Z","signature_id":"9d08db6e6be5de57ec10306b0d33b8e15ea40e6a5278009d21b49695e9d5a681","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849031038177280","occurred_at":"2026-08-21T17:10:29Z","signature_id":"5ea76d1d2bb1fdd8bb56e9971199983dd7068f7bf46c775ec6d1ed78ae8978a2","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"07eaf5ac7c17bbce04470e8a1727fe0e2aa5c26739adb10fb318a6e08fe07b87","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"fd2db70b409a9f3665fa7cca11faadb591d9872b90277565d9b8d6734403499b","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"136d54f78fc7a5cdc61527db90718c6d08f8a61c1f81807bda338e6b6d2c846f","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"88c75debdf1c34c29f7554793a0f52ff02bca445bb3ca24bd0878181de5f754e","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090612430223708160","occurred_at":"2026-08-21T01:30:11Z","signature_id":"209558f071f2f3b6cc35a68d8a5e17f16b7be58a627ef1f7287a910f7660c1c8","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090587456947621888","occurred_at":"2026-08-20T23:51:05Z","signature_id":"0fed287722e4ffab1c9718ca7b358dc431bfb749907bd990368d17d5f11fc4ae","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"fad068e6b6e4e1c995deb946f615a41fe51a51a10f0f5201c087eb71241c9966","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"da4cdd5d4991684fed1694b77cabfbc48bafe4370fec990007869d97caba68ca","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090443849187987456","occurred_at":"2026-08-20T14:20:26Z","signature_id":"dd837c8f3fd95bbca2b7933a9995248d0eb0017f33e5c8e584e0c53de04cb4b4","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090373215313465344","occurred_at":"2026-08-20T09:39:46Z","signature_id":"8299834d2efef2afff12de3e63ac69e61454ca342d920e15af7739595fdd39f9","pr_number":6589,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"b7da6f615fe5b11f0175e23954d32cde52017be4d8f9772defa4988af94e4830","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"8d04a3b31b14f6560bf5f8ba4c6c9d73c523676a77466f47e8eca833c27b2ea5","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"d4590bf5ab58e49323e9d4bbf719388cdb49a408dd121dee64f60e49573bdf2b","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"15f836f9d22cb527e0cd53b6fa63431a2ec2adc9699b9f5f0cf7b5b28f0e1858","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"85551ce531dc29c4032a21ea6ee815ac220ea9956f58c43585d642bfb6dea089","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"c262a21745c515132de6567b1de37663961ce91df589b275b9aeae9e6384874f","pr_number":6602,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":19,"runs_affected":19,"failed_runs":19,"run_impact_percent":2.996845425867508,"anchor_occurrences":19,"after_last_push_count":10,"also_in":["PROD","STG"],"trend_counts":[1,1,0,0,1,0,6,11],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":65,"prior_last_seen_at":"2026-08-22T04:44:09Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":19}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"371e2dd52ef73c97161118e4d4efe23b9ace8422b8af743704455c3d7faec23b","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260279839625216","occurred_at":"2026-08-25T14:38:17Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6630,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6677/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092248818148970496","occurred_at":"2026-08-25T13:52:37Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6677,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240300545150976","occurred_at":"2026-08-25T13:18:54Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197879035727872","occurred_at":"2026-08-25T10:30:20Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091985911028912128","occurred_at":"2026-08-24T20:28:03Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"371e2dd52ef73c97161118e4d4efe23b9ace8422b8af743704455c3d7faec23b","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"371e2dd52ef73c97161118e4d4efe23b9ace8422b8af743704455c3d7faec23b","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":14,"runs_affected":10,"failed_runs":10,"run_impact_percent":1.5772870662460567,"anchor_occurrences":14,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,13],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":15,"prior_last_seen_at":"2026-08-19T18:39:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d808a0\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-m46xgd2l9dwf\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93218d8d-7467-45c4-ba19-ca9ca2bada13\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93218d8d-7467-45c4-ba19-ca9ca2bada13\",\n      \"name\": \"93218d8d-7467-45c4-ba19-ca9ca2bada13\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:21.65358047Z\",\n      \"endTime\": \"2026-08-25T19:39:22.258875158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d808a0\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-m46xgd2l9dwf\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93218d8d-7467-45c4-ba19-ca9ca2bada13\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/93218d8d-7467-45c4-ba19-ca9ca2bada13\",\n      \"name\": \"93218d8d-7467-45c4-ba19-ca9ca2bada13\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:21.65358047Z\",\n      \"endTime\": \"2026-08-25T19:39:22.258875158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004e2c60\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-fzg7bs5ws6ct\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/444c3209-4e2e-4761-820a-0c2926e22f5c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/444c3209-4e2e-4761-820a-0c2926e22f5c\",\n      \"name\": \"444c3209-4e2e-4761-820a-0c2926e22f5c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:29.031939015Z\",\n      \"endTime\": \"2026-08-25T19:36:30.68458199Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004e2c60\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-fzg7bs5ws6ct\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/444c3209-4e2e-4761-820a-0c2926e22f5c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/444c3209-4e2e-4761-820a-0c2926e22f5c\",\n      \"name\": \"444c3209-4e2e-4761-820a-0c2926e22f5c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:29.031939015Z\",\n      \"endTime\": \"2026-08-25T19:36:30.68458199Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010fc5c0\u003e: \n    failed to create HCP cluster np-downgrade-9wkwgd: failed waiting for cluster=\"np-downgrade-9wkwgd\" in resourcegroup=\"rg-np-downgrade-9wkwgd-c6w4l726nv9w\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4ede516-845b-4e65-988d-164f84fc3105\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4ede516-845b-4e65-988d-164f84fc3105\",\n      \"name\": \"b4ede516-845b-4e65-988d-164f84fc3105\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:49.55170955Z\",\n      \"endTime\": \"2026-08-25T19:36:50.700241881Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010fc5c0\u003e: \n    failed to create HCP cluster np-downgrade-9wkwgd: failed waiting for cluster=\"np-downgrade-9wkwgd\" in resourcegroup=\"rg-np-downgrade-9wkwgd-c6w4l726nv9w\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4ede516-845b-4e65-988d-164f84fc3105\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4ede516-845b-4e65-988d-164f84fc3105\",\n      \"name\": \"b4ede516-845b-4e65-988d-164f84fc3105\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:49.55170955Z\",\n      \"endTime\": \"2026-08-25T19:36:50.700241881Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"bdffa9ec41658c3c0b2fef63b1f22f6a6742e6c4e837e0fea114368b4612c51e","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"4ca0d90c5accc251f79a548ac4f235fa37708ec7a5a06deeb65a77bd99992647","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"a5e9145697fe5d26a793f74527aa00e2ec11f8e0d29fd25f899226a0c8ca6786","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269882870599680","occurred_at":"2026-08-25T15:16:27Z","signature_id":"f337aa7b62ef749e20f60dfec4dfcb7a4238a15c3da0837817bfff818d840f30","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"77a4fccb98689590052e444032f6e0b2a81fc76d19cc8dedaba7314fbe9ef5b2","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"f3706b7b68e9a79a6664d2f4e9bb40f34471b0eda04d386d956f5c3d42464cb6","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","signature_id":"020633b5339d75f1df2cb64c5d4e04f6ce5d65297c8f1dc716fac948850492b0","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"50cfe241bd776c63ead0185c89db291d20bfefd14fa4462a1eff23a8bb7280f2","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"d6949fb19ef2eea4676f8abc14bdf9d0083de17b97edd1d98d989d65f25a63ef","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"841f5d6b1662eac99a6cc527de5c5a356bc8edcef21537485d177eab3f47e0b8","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"f7a0795ccf41d2868c0062e96d14bdc47a40e0b4d69128fb529e4080a9cfca2a","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"37180a3567b7ba4c19f170d6ea685b79ea6d88f1f355e96431dbe0380eee9693","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"ab0efe968c3af5d317e92a58ea9e41044d2452ac55e715d3c4270447bf41e742","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"c7d630f0a28c9b8d1aeb7eeaccfb13c753fc80a06498686e5dd2af7ee7790d98","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":14,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.2618296529968454,"anchor_occurrences":14,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,10,4],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001022220\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-dm2lv9wtz9pd\" to finish creating: GET https://rp.j9302656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0f240228-e711-46ec-8078-6d29993f3656\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0f240228-e711-46ec-8078-6d29993f3656\",\n      \"name\": \"0f240228-e711-46ec-8078-6d29993f3656\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T18:54:14.698555112Z\",\n      \"endTime\": \"2026-08-25T19:13:18.75896335Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001022220\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-dm2lv9wtz9pd\" to finish creating: GET https://rp.j9302656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0f240228-e711-46ec-8078-6d29993f3656\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0f240228-e711-46ec-8078-6d29993f3656\",\n      \"name\": \"0f240228-e711-46ec-8078-6d29993f3656\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T18:54:14.698555112Z\",\n      \"endTime\": \"2026-08-25T19:13:18.75896335Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00041c4e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6bdpbrgzdzc8\" to finish creating: GET https://rp.j9832320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccc072ba-d2dd-4a86-b29f-6086f60cb482\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccc072ba-d2dd-4a86-b29f-6086f60cb482\",\n      \"name\": \"ccc072ba-d2dd-4a86-b29f-6086f60cb482\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T17:41:06.665554524Z\",\n      \"endTime\": \"2026-08-25T18:00:10.653150122Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00041c4e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6bdpbrgzdzc8\" to finish creating: GET https://rp.j9832320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccc072ba-d2dd-4a86-b29f-6086f60cb482\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccc072ba-d2dd-4a86-b29f-6086f60cb482\",\n      \"name\": \"ccc072ba-d2dd-4a86-b29f-6086f60cb482\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T17:41:06.665554524Z\",\n      \"endTime\": \"2026-08-25T18:00:10.653150122Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9eb40\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-nmp69vm78h4k\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/584d4084-30a5-4233-a069-f2a0cf2b9043\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/584d4084-30a5-4233-a069-f2a0cf2b9043\",\n      \"name\": \"584d4084-30a5-4233-a069-f2a0cf2b9043\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:44:03.375670894Z\",\n      \"endTime\": \"2026-08-25T17:03:09.343723737Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9eb40\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-nmp69vm78h4k\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/584d4084-30a5-4233-a069-f2a0cf2b9043\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/584d4084-30a5-4233-a069-f2a0cf2b9043\",\n      \"name\": \"584d4084-30a5-4233-a069-f2a0cf2b9043\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:44:03.375670894Z\",\n      \"endTime\": \"2026-08-25T17:03:09.343723737Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092311337039302656","occurred_at":"2026-08-25T18:01:04Z","signature_id":"970a4e186dca3e58eb5fc30dda68bc89be799ddbb9763794ffcc3be928d20bb8","pr_number":6639,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092295016859832320","occurred_at":"2026-08-25T16:56:19Z","signature_id":"0b8690595b1dfed6e36bd3e52f32e3625ed1e752008ebb3edcf9215308afc984","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"473e68c054364e4d1143001e092d3c5f19b78d56a2f0385244813ef7b9b36807","pr_number":6679,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092136706911244288","occurred_at":"2026-08-25T06:27:13Z","signature_id":"ab35ad7b76b266586beb627cd857234f4deca4448950461a1483290474e1d37e","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"87055b770927de3226a00a72e6a1e8f425c4181b7305ba9fd94502a36df5fec7","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"e5919a5aa6e234b2c6b16539e4375d2ff9f7724d31d00cb9d8f2c6f3bd3b38ad","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"4bba4cf482e3fa39dfa2ce3e799d7eb2c477fcc7e6ad0b0350124af0d131c053","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"5a1d95e1ce2afb3cd1318374aea17f4a6e4f5d30ef03f9f5a0cc441e9043a771","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"28a8b869f35a9cd1794a8e197fc2255f15a11da6d73b85987e232efa1871d3dc","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"2cae48dc9a7ac77e5a8ac54a32e19e04d69dc07b4bbfdffac675fbb83525b59a","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"51d9d625d26042275bccf29b990ed16b6ab0c82bab381da48ae1943a65da776a","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"57e93beced19ad75ed8576f3f0efd935e4fd8431d2ed07db01840f4e2fefa928","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"9137a39c80626f35b65cbd5b530df9d8f5ceed8a3ed523d23d20fe02b5217e4f","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","signature_id":"0119ef26d501844a6f69a87b74e13b68dec77db9c3af742f0d6e0ca5c74e1853","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4984p9m7f6lc\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4984p9m7f6lc, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":12,"runs_affected":12,"failed_runs":12,"run_impact_percent":1.8927444794952681,"anchor_occurrences":12,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[0,2,1,4,2,0,1,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":22,"prior_last_seen_at":"2026-08-22T14:52:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":8},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:153]: failed to scale up node pool np-update-nodes from 2 to 3 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0024223c0\u003e: \n    failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-gld8nsfdc76t\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-gld8nsfdc76t, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:153]: failed to scale up node pool np-update-nodes from 2 to 3 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0024223c0\u003e: \n    failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-gld8nsfdc76t\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-gld8nsfdc76t, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:208]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0033fc180\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-jz5t9hfw7w26\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jz5t9hfw7w26, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:208]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0033fc180\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-jz5t9hfw7w26\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jz5t9hfw7w26, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:208]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00396d0e0\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4984p9m7f6lc\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4984p9m7f6lc, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:208]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00396d0e0\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4984p9m7f6lc\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4984p9m7f6lc, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092313647266140160","occurred_at":"2026-08-25T18:10:21Z","signature_id":"b81c2e95d57f9e709b06188bf80fa3ab504036a41d889fef881537e3c942592c","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"99c112613fb0611797f784fb621f4a92a4be1186e7f636e286eaba4372997bdb","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","signature_id":"dfb206f2ba6bc7bc3ce13c32e974065647bae3407bafdce87be1e26f94b6936e","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091176636450869248","occurred_at":"2026-08-22T14:52:16Z","signature_id":"2279f0e21f0118ec4cfb855020d1fb23afa7a8053b7e28a26a3e788f00eff3ba","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090989679758282752","occurred_at":"2026-08-22T02:29:22Z","signature_id":"79acbfdba47e0e4a6c76392e334f88cd638d51646a498b2bfb66f8deaf94d4c5","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090884661910704128","occurred_at":"2026-08-21T19:32:04Z","signature_id":"02f8cf971ee2086a5a5c0f3e34976f3baeeca58a897d042c67c691ba1f89b42d","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090833794218070016","occurred_at":"2026-08-21T16:09:56Z","signature_id":"031dc05be95ed92f826a45611c2e85f662a26b57a5843c7a19763beaeca101c2","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817534927310848","occurred_at":"2026-08-21T15:05:19Z","signature_id":"d3641496165ea9fc11207fde332bc727e7add4426bbba9ed882ee06f8b3ed6aa","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090766839033368576","occurred_at":"2026-08-21T11:43:53Z","signature_id":"ce181594deb79c253e35d008cf1bd9fa400dffdb87c80b1827c3a6d4e05ad422","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090355263239360512","occurred_at":"2026-08-20T08:28:26Z","signature_id":"10a5ec416a1194f12713357f6f75e0d897f7d7fbfc371d611a8f593beff77b00","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090181452632166400","occurred_at":"2026-08-19T20:57:46Z","signature_id":"5cf10238ae32cb119c8dedd783e284c2ab85fb7d2d666794b8ad8ff1902d5a2c","pr_number":6572,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"67a86f16b7c04eb1bdd0def775c5fa85beab5151539ba43c3c263ef5583b993e","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a871b3ae1e8360285972402ef71b07429b5d17f69841894ee06b053d348eb8de","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=1)","occurrences":11,"runs_affected":11,"failed_runs":11,"run_impact_percent":1.7350157728706626,"anchor_occurrences":11,"after_last_push_count":10,"trend_counts":[3,1,1,4,0,0,2,3],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":30,"prior_last_seen_at":"2026-08-21T16:05:36Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=1)","occurrences":7},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=2)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/management/output (stamp=1)","occurrences":3}],"full_error_samples":["time=2026-08-25T18:56:45.688Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 description=\"Step output\\n  Kind: ARM\\n  Template: ../../dev-infrastructure/templates/output-mgmt.bicep\\n  Parameters: ../../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-25T18:56:46.086Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1\ntime=2026-08-25T18:56:46.474Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 deployment=74b140f8b533b36c1cdb52a5204bd298a1350ffd05ecac7c751c39ee014fab07 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3363840-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F74b140f8b533b36c1cdb52a5204bd298a1350ffd05ecac7c751c39ee014fab07\ntime=2026-08-25T18:56:46.527Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3363840-mgmt-1/providers/Microsoft.Resources/deployments/74b140f8b533b36c1cdb52a5204bd298a1350ffd05ecac7c751c39ee014fab07/operationStatuses/08584139226791753333\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentNotFound\\\",\\n    \\\"message\\\": \\\"Deployment '74b140f8b533b36c1cdb52a5204bd298a1350ffd05ecac7c751c39ee014fab07' could not be found.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-25T15:21:22.056Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 description=\"Step output\\n  Kind: ARM\\n  Template: ../../dev-infrastructure/templates/output-mgmt.bicep\\n  Parameters: ../../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-25T15:21:22.495Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1\ntime=2026-08-25T15:21:22.967Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 deployment=5e98ce680849833c628986a2e68efe7c2667cb97ae72f93416c33a4c5e66f0ac portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6676864-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F5e98ce680849833c628986a2e68efe7c2667cb97ae72f93416c33a4c5e66f0ac\ntime=2026-08-25T15:21:23.080Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6676864-mgmt-1/providers/Microsoft.Resources/deployments/5e98ce680849833c628986a2e68efe7c2667cb97ae72f93416c33a4c5e66f0ac/operationStatuses/08584139356027184199\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentNotFound\\\",\\n    \\\"message\\\": \\\"Deployment '5e98ce680849833c628986a2e68efe7c2667cb97ae72f93416c33a4c5e66f0ac' could not be found.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-25T14:41:26.287Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=2 description=\"Step output\\n  Kind: ARM\\n  Template: ../../dev-infrastructure/templates/output-mgmt.bicep\\n  Parameters: ../../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-25T14:41:26.667Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=2\ntime=2026-08-25T14:41:27.045Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=2 deployment=3b44e853e3fa2f5ba695fe1eeadf1d38b957019eefef20ca1a49aa6b0c7a6f2b portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6436096-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F3b44e853e3fa2f5ba695fe1eeadf1d38b957019eefef20ca1a49aa6b0c7a6f2b\ntime=2026-08-25T14:41:27.116Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6436096-mgmt-2/providers/Microsoft.Resources/deployments/3b44e853e3fa2f5ba695fe1eeadf1d38b957019eefef20ca1a49aa6b0c7a6f2b/operationStatuses/08584139379985620464\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentNotFound\\\",\\n    \\\"message\\\": \\\"Deployment '3b44e853e3fa2f5ba695fe1eeadf1d38b957019eefef20ca1a49aa6b0c7a6f2b' could not be found.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314609523363840","occurred_at":"2026-08-25T18:14:10Z","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260182896676864","occurred_at":"2026-08-25T14:37:54Z","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","pr_number":6661,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253020946436096","occurred_at":"2026-08-25T14:09:13Z","signature_id":"3b416c8626d510d6d228f2b4d85a59b118ef1a435941087523bd321e4dac7444","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091954836298797056","occurred_at":"2026-08-24T18:15:46Z","signature_id":"0ec7efd378b135d7fa85cb93bcc2f610fe36f768df0ac4e2e0fe6122f4092f85","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928527292076032","occurred_at":"2026-08-24T16:32:12Z","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6650/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090832700310032384","occurred_at":"2026-08-21T16:05:36Z","signature_id":"0ec7efd378b135d7fa85cb93bcc2f610fe36f768df0ac4e2e0fe6122f4092f85","pr_number":6650,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090730615539240960","occurred_at":"2026-08-21T09:19:57Z","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090717629881782272","occurred_at":"2026-08-21T08:28:21Z","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090618928278540288","occurred_at":"2026-08-21T01:56:08Z","signature_id":"0ec7efd378b135d7fa85cb93bcc2f610fe36f768df0ac4e2e0fe6122f4092f85","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090541078221426688","occurred_at":"2026-08-20T20:46:48Z","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6515/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090152370880122880","occurred_at":"2026-08-19T19:02:11Z","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","pr_number":6515,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bc64b77873a26db9d4fec58f70db7822ca134ba2fdf43abf022203397f106424","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":11,"runs_affected":10,"failed_runs":10,"run_impact_percent":1.5772870662460567,"anchor_occurrences":11,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,11],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":10,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010cc300\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-j56jzgdfb84c\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/152cdec3-f7df-4add-8bd0-f0c465a1cce4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/152cdec3-f7df-4add-8bd0-f0c465a1cce4\",\n      \"name\": \"152cdec3-f7df-4add-8bd0-f0c465a1cce4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:22:55.074048426Z\",\n      \"endTime\": \"2026-08-25T19:42:02.420103182Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010cc300\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-j56jzgdfb84c\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/152cdec3-f7df-4add-8bd0-f0c465a1cce4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/152cdec3-f7df-4add-8bd0-f0c465a1cce4\",\n      \"name\": \"152cdec3-f7df-4add-8bd0-f0c465a1cce4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:22:55.074048426Z\",\n      \"endTime\": \"2026-08-25T19:42:02.420103182Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00128a380\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-7p5xtj25zdgq\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02c1dd80-6bb6-452a-8773-44a463672ab0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02c1dd80-6bb6-452a-8773-44a463672ab0\",\n      \"name\": \"02c1dd80-6bb6-452a-8773-44a463672ab0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:23:27.029566085Z\",\n      \"endTime\": \"2026-08-25T19:42:33.615773728Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00128a380\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-7p5xtj25zdgq\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02c1dd80-6bb6-452a-8773-44a463672ab0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02c1dd80-6bb6-452a-8773-44a463672ab0\",\n      \"name\": \"02c1dd80-6bb6-452a-8773-44a463672ab0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:23:27.029566085Z\",\n      \"endTime\": \"2026-08-25T19:42:33.615773728Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-hrhcgp\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dce540\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-hrhcgp: failed waiting for cluster=\"cluster-zstream-5-0-hrhcgp\" in resourcegroup=\"rg-zstream-upgrade-5-0-78mnn44hkdrc\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b95d4f1a-9b50-422c-9b91-4f716507dd2d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b95d4f1a-9b50-422c-9b91-4f716507dd2d\",\n      \"name\": \"b95d4f1a-9b50-422c-9b91-4f716507dd2d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:27.785135797Z\",\n      \"endTime\": \"2026-08-25T16:40:28.593240311Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-hrhcgp\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dce540\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-hrhcgp: failed waiting for cluster=\"cluster-zstream-5-0-hrhcgp\" in resourcegroup=\"rg-zstream-upgrade-5-0-78mnn44hkdrc\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b95d4f1a-9b50-422c-9b91-4f716507dd2d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b95d4f1a-9b50-422c-9b91-4f716507dd2d\",\n      \"name\": \"b95d4f1a-9b50-422c-9b91-4f716507dd2d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:27.785135797Z\",\n      \"endTime\": \"2026-08-25T16:40:28.593240311Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"e7d19e4ec13af075778836b0509e48aae2d3aa918859115e31905f9216f6eda2","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"2a3c305e2e7b889cc83bd789e19181806498ad35731017cdff9ccc00c9807367","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"04200eec52043650488776ea91f3a75dc9e4bb6cd15be64da7bc9e97ceff3c5e","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","signature_id":"5aee83e4a720dd3ebb6047456af4c05c8c39902dcd0a2d8df1e2007ebdf2f0c2","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","signature_id":"b2eebc2a537ba9b4668bce19ae91591fac3e22a5f8939d98e9175fbce444f838","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"0df35959863c4a18d18241249045541a76f7fb86ff7bc0c329f8da11e7941ea4","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"5ae8b49cfd62a1fd2be6d40eb9b8f7b2df783c47f66002e18ea549ef9fae6614","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"8264131fc2b45d2f5ca5d9936dd76c0aeb91f30f562df92f37bbc56a238e6d26","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"cfec1836d28d37c357fa4cdaaa7a7e2cfa94f1863cd3c937f9210fab49a6b769","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"b5ee1f8ad40d4305b4dfae8b6e148d13794c07d1d9f1c5fc72025bff8e810890","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"cf304d6dfff36c6441a2fb53d72be066b0745f911a7fb1567e495cd2a83bfd4e","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":11,"runs_affected":9,"failed_runs":9,"run_impact_percent":1.4195583596214512,"anchor_occurrences":11,"after_last_push_count":3,"also_in":["PROD"],"trend_counts":[1,0,0,1,0,0,0,10],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":17,"prior_last_seen_at":"2026-08-21T06:13:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00131c880\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-bxjjlgfmc9db\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3a4169eb-2032-480c-8f15-d84b8528bedf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3a4169eb-2032-480c-8f15-d84b8528bedf\",\n      \"name\": \"3a4169eb-2032-480c-8f15-d84b8528bedf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:23:19.200664013Z\",\n      \"endTime\": \"2026-08-25T19:42:23.611070686Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00131c880\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-bxjjlgfmc9db\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3a4169eb-2032-480c-8f15-d84b8528bedf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3a4169eb-2032-480c-8f15-d84b8528bedf\",\n      \"name\": \"3a4169eb-2032-480c-8f15-d84b8528bedf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:23:19.200664013Z\",\n      \"endTime\": \"2026-08-25T19:42:23.611070686Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012ea320\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-h5k4cp2j9wxp\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36f15d29-956b-4c81-a94e-9ff3c38026e3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36f15d29-956b-4c81-a94e-9ff3c38026e3\",\n      \"name\": \"36f15d29-956b-4c81-a94e-9ff3c38026e3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:46.898858572Z\",\n      \"endTime\": \"2026-08-25T19:36:50.642648068Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012ea320\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-h5k4cp2j9wxp\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36f15d29-956b-4c81-a94e-9ff3c38026e3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36f15d29-956b-4c81-a94e-9ff3c38026e3\",\n      \"name\": \"36f15d29-956b-4c81-a94e-9ff3c38026e3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:46.898858572Z\",\n      \"endTime\": \"2026-08-25T19:36:50.642648068Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-9v2fm7\" with version 4.22.10 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000abc760\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-9v2fm7: failed waiting for cluster=\"cluster-zstream-4-22-9v2fm7\" in resourcegroup=\"rg-zstream-upgrade-4-22-qzxxgjsj26qb\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02d2964a-7a05-4345-980d-51e9bdf2ffaf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02d2964a-7a05-4345-980d-51e9bdf2ffaf\",\n      \"name\": \"02d2964a-7a05-4345-980d-51e9bdf2ffaf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:54:16.06655817Z\",\n      \"endTime\": \"2026-08-25T15:13:19.307227089Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-9v2fm7\" with version 4.22.10 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000abc760\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-9v2fm7: failed waiting for cluster=\"cluster-zstream-4-22-9v2fm7\" in resourcegroup=\"rg-zstream-upgrade-4-22-qzxxgjsj26qb\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02d2964a-7a05-4345-980d-51e9bdf2ffaf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/02d2964a-7a05-4345-980d-51e9bdf2ffaf\",\n      \"name\": \"02d2964a-7a05-4345-980d-51e9bdf2ffaf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:54:16.06655817Z\",\n      \"endTime\": \"2026-08-25T15:13:19.307227089Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"7920aacad73a762e7cb4200cc3f935f23f19001cc15e0d371213e2d503e50bf5","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"c1c3211636089d001497e52a1029d888d6af4966bdcf5a039a78f8eb9fff0edc","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","signature_id":"c3b3ef09c20cc547bca5325c7f41e4dd0c4a174e6693ff2418f6a450ed448c6b","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092244166900715520","occurred_at":"2026-08-25T13:34:16Z","signature_id":"ba2c82cf6ae3a73e48ea720374e43d7b38d997b258802c349e2dac2f93c95ba4","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"941ca1754672c53670ae6f9235025ce03ee81e89e2ab50e5ebdc653a9104513b","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"bd627bb7dd0c1e61533ffcaff0de1677c51817a542e05bd6bac955235f59715c","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"d170f04f7cb6344abd769b118fc7f073e431f9b518426774c88ff7532f8e723d","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"036e61e55afe3d161610066648e9dacc1c80f9cbd63ca3be6cf42026860462b7","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"2a0367cc0e9683bd02dfae7b534805fb13ea99e1bfc239a750d74f7845bd9014","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"6a014dc04f005c0ce2a8189b9d8c8434c9d29f69cb03bf76fae5f1ea1b289999","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090683781416488960","occurred_at":"2026-08-21T06:13:51Z","signature_id":"4ca248892099e2c2d4e4a976050bdfef3c0ae82bdef9070cafe3f93ce794316c","pr_number":6519,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6cf0bbf9b7d17f41cb34ef56e829ec2324222ca77c76bbfc427185392db35615","failure_pattern":"Error from server (ServiceUnavailable): the server is currently unable to handle the request)","search_query":"Error from server (ServiceUnavailable): the server is currently unable to handle the request)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":11,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.9463722397476341,"anchor_occurrences":11,"after_last_push_count":0,"trend_counts":[0,0,0,3,6,2,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-23T03:43:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS, private ingress, and private KeyVault and verify all are operational","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:172]: Timed out after 3623.396s.\nKAS should be reachable from VM via internal LB\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:170 with:\nkubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nError from server (ServiceUnavailable): the server is currently unable to handle the request)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:172]: Timed out after 3623.396s.\nKAS should be reachable from VM via internal LB\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:170 with:\nkubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nError from server (ServiceUnavailable): the server is currently unable to handle the request)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:203]: Timed out after 3630.452s.\nKAS should be reachable from VM via internal LB\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:201 with:\nkubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nError from server (ServiceUnavailable): the server is currently unable to handle the request)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:203]: Timed out after 3630.452s.\nKAS should be reachable from VM via internal LB\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:201 with:\nkubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nError from server (ServiceUnavailable): the server is currently unable to handle the request)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:172]: Timed out after 1221.685s.\nKAS should be reachable from VM via internal LB\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:170 with:\nkubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nError from server (ServiceUnavailable): the server is currently unable to handle the request)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:172]: Timed out after 1221.685s.\nKAS should be reachable from VM via internal LB\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:170 with:\nkubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nError from server (ServiceUnavailable): the server is currently unable to handle the request)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091370674797940736","occurred_at":"2026-08-23T03:43:19Z","signature_id":"a2d132dfde07f3391ff7b4a1c2de45cb2c603d801bc82cc4a5fbbe7df390ef7e","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091370674797940736","occurred_at":"2026-08-23T03:43:19Z","signature_id":"d2285d363ed1a1e560ce21be276ca5a6521530a681d4f19d7cb7f03800b0b0a1","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091230843602210816","occurred_at":"2026-08-22T18:27:40Z","signature_id":"b7d765b338677f9132d9cff7a564a334e39b7d73a284b7fd67bdad2484900a37","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091230843602210816","occurred_at":"2026-08-22T18:27:40Z","signature_id":"e93aca990e9358976e512f671aa4caa5d2bcaeab27afcaa4ca5fdcddc64ad356","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091176636450869248","occurred_at":"2026-08-22T14:52:16Z","signature_id":"69c3f7216d3016ebb9d2fb04a005788dacee246d0f78bb02875390b4e0edcae0","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091176636450869248","occurred_at":"2026-08-22T14:52:16Z","signature_id":"d8865b61f871581c8d18c544a842a91b67cbd9285ac45828a068750fdf612811","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090989679758282752","occurred_at":"2026-08-22T02:29:22Z","signature_id":"3822fa1effd49ba7c1cdd3d0033f308a6711f286fd0ceab8d0d8b7b1e0be414a","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090989679758282752","occurred_at":"2026-08-22T02:29:22Z","signature_id":"d5ea982e91fda6f482df8e6cf7fbadfda4c5f9261aef6f580c5e75ddca0ed96d","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"d5ea982e91fda6f482df8e6cf7fbadfda4c5f9261aef6f580c5e75ddca0ed96d","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090884661910704128","occurred_at":"2026-08-21T19:32:04Z","signature_id":"b3f9577b121e1d710a77d5ca9c61cd869d4fd50ea68e082da03b5517331dd8cd","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090884661910704128","occurred_at":"2026-08-21T19:32:04Z","signature_id":"b3f9577b121e1d710a77d5ca9c61cd869d4fd50ea68e082da03b5517331dd8cd","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":9,"runs_affected":9,"failed_runs":9,"run_impact_percent":1.4195583596214512,"anchor_occurrences":9,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,2,6],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":9,"prior_last_seen_at":"2026-08-22T00:32:00Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":9}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T19:49:51Z\n  Ended:       2026-08-25T19:57:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8405376-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8405376-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T19:50:28Z\n  Ended:       2026-08-25T19:56:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7285632-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7285632-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T19:13:57Z\n  Ended:       2026-08-25T19:41:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6140160-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6140160-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"57c5ec60d7b5f08bc41aec5dff584edf947ed9bf0d23769b021eb635a33b3d21","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"0ca8bc99c3c40c8ea160690445d9fbca4420866493398175f96a3e330d91ed47","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092313647266140160","occurred_at":"2026-08-25T18:10:21Z","signature_id":"42ec01f8964f22f9352e4ea60e7c8934ff52d4ff3ff5e12af66623897e7c4751","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261270257405952","occurred_at":"2026-08-25T14:42:10Z","signature_id":"a433ce52553f210f473828de792bdb37ba8e12f31a35d7427eb780d885009233","pr_number":6614,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"f9b1dbaf8f8300b20f00ab73fece95cb317fd0f48d3a396dbbe2c51c020ad097","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092188089500831744","occurred_at":"2026-08-25T09:29:48Z","signature_id":"6db69bc2fe95a94bfd8232f25627f1581b6967e549eb3db637637bd4cef65f38","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932359686361088","occurred_at":"2026-08-24T16:54:34Z","signature_id":"a3d3dfe16c45487c7de4833dcff7c72d504e34a1b391992ceada6266956233ad","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091921367619342336","occurred_at":"2026-08-24T16:10:41Z","signature_id":"d4e52f6e7208e4a023c43c3ea9a95264341c256a3dcba5d703516c2b2c81f97b","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090960143322714112","occurred_at":"2026-08-22T00:32:00Z","signature_id":"41cd7d7000b0d0b522b6e180f5feaf065d54e9a6666f9085acc644c179c02e0c","pr_number":6639,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":9,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.2618296529968454,"anchor_occurrences":9,"after_last_push_count":1,"also_in":["PROD","STG"],"trend_counts":[3,0,0,0,0,0,0,9],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":13,"prior_last_seen_at":"2026-08-18T18:57:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007c92e0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8wz885flg8mk\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f6cc866-fb13-4391-b5e4-4ef1f14b4973\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f6cc866-fb13-4391-b5e4-4ef1f14b4973\",\n      \"name\": \"9f6cc866-fb13-4391-b5e4-4ef1f14b4973\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:24:58.553705193Z\",\n      \"endTime\": \"2026-08-25T19:44:03.525820627Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007c92e0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8wz885flg8mk\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f6cc866-fb13-4391-b5e4-4ef1f14b4973\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f6cc866-fb13-4391-b5e4-4ef1f14b4973\",\n      \"name\": \"9f6cc866-fb13-4391-b5e4-4ef1f14b4973\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:24:58.553705193Z\",\n      \"endTime\": \"2026-08-25T19:44:03.525820627Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f71140\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7fhrl8nsqhmm\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\",\n      \"name\": \"9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:28.727590226Z\",\n      \"endTime\": \"2026-08-25T16:40:28.622162047Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f71140\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7fhrl8nsqhmm\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\",\n      \"name\": \"9ec8537f-8ee3-4b4f-990c-d50e9ecb7497\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:28.727590226Z\",\n      \"endTime\": \"2026-08-25T16:40:28.622162047Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013f6760\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-8qxvb7djrfqs\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ef3c075-0505-498f-acb7-bec851217620\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ef3c075-0505-498f-acb7-bec851217620\",\n      \"name\": \"0ef3c075-0505-498f-acb7-bec851217620\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:13:28.807201061Z\",\n      \"endTime\": \"2026-08-25T16:32:29.809496877Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013f6760\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-8qxvb7djrfqs\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ef3c075-0505-498f-acb7-bec851217620\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ef3c075-0505-498f-acb7-bec851217620\",\n      \"name\": \"0ef3c075-0505-498f-acb7-bec851217620\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:13:28.807201061Z\",\n      \"endTime\": \"2026-08-25T16:32:29.809496877Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"acd72cf8b7bb2f45dae55eed6a696c8c2095d677ca82dcc3ccb5ef413f64290e","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"8736192f6f4f645775b5c68f46d7ee7ddc0264e06e6677e85bab2a90cd28f2bc","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","signature_id":"d90c9eedcebd8ebee811c56262835a367e830ea3ef985a6b9e0519703d7c4512","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"bccfe75fe11d0ea8f7728479404e3d27d3b6a70c50e8e5d0793d665e9d222fbf","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"01b87695dd9b2116a68fd8b6e0c91abd3e780e11edb0177bfb654801580cf72d","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"1f29965bd436288d269b00ceb20f18dc47e327d03580c8db54436fbf801d2664","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"6aa2e7ee830b9fa0230c736bace065dc0ed283e1350606e8f5015746076bf57f","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"ddfd07ad7c5094cf706dd735ad3c3aed8ba35bd4d8f19d89a430d7152ef2a389","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"bbe4d66acf643736fa2f7c6c58f4e7114799ee91a201d8f3333899113347be58","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f88ac713b613a590b2fe8fa69068a61ed7140b6fed609b9849143fda8fd9e327","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":9,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.7886435331230284,"anchor_occurrences":9,"after_last_push_count":0,"trend_counts":[0,6,3,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-20T03:19:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011680d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-z85t4l4vlbzm\" to finish deleting: GET https://rp.j0378240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5056cdf-a481-4533-9ce3-097744d6b413\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5056cdf-a481-4533-9ce3-097744d6b413\",\n      \"name\": \"b5056cdf-a481-4533-9ce3-097744d6b413\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T04:16:44.794311199Z\",\n      \"endTime\": \"2026-08-20T04:40:52.757543513Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011680d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-z85t4l4vlbzm\" to finish deleting: GET https://rp.j0378240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5056cdf-a481-4533-9ce3-097744d6b413\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5056cdf-a481-4533-9ce3-097744d6b413\",\n      \"name\": \"b5056cdf-a481-4533-9ce3-097744d6b413\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T04:16:44.794311199Z\",\n      \"endTime\": \"2026-08-20T04:40:52.757543513Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019f6078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-dvmg5tnhn7bl\" to finish deleting: GET https://rp.j0378240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/59a1bde3-3626-4d39-bb21-86f7656900a9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/59a1bde3-3626-4d39-bb21-86f7656900a9\",\n      \"name\": \"59a1bde3-3626-4d39-bb21-86f7656900a9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T04:15:36.007694317Z\",\n      \"endTime\": \"2026-08-20T04:39:42.78156064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019f6078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-dvmg5tnhn7bl\" to finish deleting: GET https://rp.j0378240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/59a1bde3-3626-4d39-bb21-86f7656900a9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/59a1bde3-3626-4d39-bb21-86f7656900a9\",\n      \"name\": \"59a1bde3-3626-4d39-bb21-86f7656900a9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T04:15:36.007694317Z\",\n      \"endTime\": \"2026-08-20T04:39:42.78156064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014d2078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-j9kfk6656slq\" to finish deleting: GET https://rp.j3139200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/758cadb0-636e-4143-b1b5-5771c89ce22a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/758cadb0-636e-4143-b1b5-5771c89ce22a\",\n      \"name\": \"758cadb0-636e-4143-b1b5-5771c89ce22a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T02:09:52.194794521Z\",\n      \"endTime\": \"2026-08-20T02:33:57.777012859Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014d2078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-j9kfk6656slq\" to finish deleting: GET https://rp.j3139200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/758cadb0-636e-4143-b1b5-5771c89ce22a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/758cadb0-636e-4143-b1b5-5771c89ce22a\",\n      \"name\": \"758cadb0-636e-4143-b1b5-5771c89ce22a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T02:09:52.194794521Z\",\n      \"endTime\": \"2026-08-20T02:33:57.777012859Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090277524570378240","occurred_at":"2026-08-20T03:19:31Z","signature_id":"07359dd26fae5073ad07b23e2966b25ea8b7f336630101293a6a95a60847a6b1","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090277524570378240","occurred_at":"2026-08-20T03:19:31Z","signature_id":"8d0ad4d2f288a6b561e87de5504ec0054fced4d300c4b16a0b8de371406bc244","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"2650886f97daa2bac726adb3cee2f77ef1eb8b00325d923a67d6bb25b6830c20","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090222553485283328","occurred_at":"2026-08-19T23:41:05Z","signature_id":"22848fa078d0328825e14eed64b734b1953af1ba5f960122db077972a9bf624c","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090222553485283328","occurred_at":"2026-08-19T23:41:05Z","signature_id":"99ed3252f28c8c1c60792d87d919d200608c9ec9477676f42ad1850bb1179ba1","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090222553485283328","occurred_at":"2026-08-19T23:41:05Z","signature_id":"db7bf377a9e008ff66c454905ea366296951bc25fbb317aa85375f3caeab10d2","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090192872639827968","occurred_at":"2026-08-19T21:43:09Z","signature_id":"1d4cf42db36be45b0ad4ad46fd8aef7274ba52884b1241cf883d6084345a92a8","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090192872639827968","occurred_at":"2026-08-19T21:43:09Z","signature_id":"5527339989c73b7db0113c00563acb3179d3cd3ef9058e6068498b51fbc57c43","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090157821952790528","occurred_at":"2026-08-19T19:23:52Z","signature_id":"05c7f175202f00963e3302472b038a8f4574705534ee660adf9aa07ee10b0c98","pr_number":6612,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.2618296529968454,"anchor_occurrences":8,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,1,6],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":29,"prior_last_seen_at":"2026-08-20T22:52:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":8}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d3766fe34f03f22876c777e51a7be1734ee728533bfb3f28e111bf0b35fcad1b","failure_pattern":"UpdateHCPCluster20260630 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-72g5866h88vj\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-72g5866h88vj, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.2618296529968454,"anchor_occurrences":8,"after_last_push_count":1,"trend_counts":[0,3,1,2,0,0,1,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":61,"prior_last_seen_at":"2026-08-21T15:05:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":8}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00110af90\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-72g5866h88vj\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-72g5866h88vj, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00110af90\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-72g5866h88vj\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-72g5866h88vj, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00132c390\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-cflktt657pch\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-cflktt657pch, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00132c390\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-cflktt657pch\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-cflktt657pch, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00154c060\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-hhjbvcghxvgq\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-hhjbvcghxvgq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00154c060\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-hhjbvcghxvgq\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-hhjbvcghxvgq, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"44f45bd7c197771dae334f78215fd7f0e9a8f7b495a48ad1be4b5f40460c338c","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","signature_id":"78869f45f7134e749985fc5b23dff71b456538a41d70734a716f1e608670ab6c","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817534927310848","occurred_at":"2026-08-21T15:05:19Z","signature_id":"f8c1f9d6e4a842b1616d9cf1c6c5f32baae9995231c4a073b145ef14db36466d","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090766839033368576","occurred_at":"2026-08-21T11:43:53Z","signature_id":"73c65419fc618c891d660bbf76f7851fce8c4689b50628554d1a8c94e626ef6a","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"f4dcc26904b6619c31c3833aeb00b8bdbc202a180a5f1e9f30f427ca0234c47d","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090126361980047360","occurred_at":"2026-08-19T17:18:51Z","signature_id":"fb4032a22ac181febfd936b28ac5ea4e27236ca4b44c9cca6e758fa816729066","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"7c9a82bf8c17109250ce7aecb9c10e968aa4ef37dcbf68fe936e3a7e20e58b13","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"3eebc351608e267dcac0b25afc008409f4090bfbafdbe3d7c34eb3432bd2f54e","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.2618296529968454,"anchor_occurrences":8,"after_last_push_count":5,"trend_counts":[2,2,2,0,2,0,1,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":36,"prior_last_seen_at":"2026-08-22T08:48:46Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":8}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T08:19:22Z\n  Ended:       2026-08-25T08:37:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6292864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6292864/providers/microsoft.monitor/accounts/hcps-j6292864\", namespace=\"ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8f2abcc4-a7f6-4094-8ae9-551ab00643de\"\n  Description: Pod ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-25T08:19:22Z\n  Ended:       2026-08-25T08:36:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6292864-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6292864/providers/microsoft.monitor/accounts/hcps-j6292864\", namespace=\"ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8f2abcc4-a7f6-4094-8ae9-551ab00643de\"\n  Description: Pod ocm-arohcpci01-2sdfp9pn2ipl52p137025ph1c09o5rmr-m7j4d7m1x6s1n9o/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T18:02:20Z\n  Ended:       2026-08-24T18:38:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4cebd923-31c4-4c0a-b568-63bf1a281b9a\"\n  Description: Pod ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T18:03:20Z\n  Ended:       2026-08-24T18:38:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4cebd923-31c4-4c0a-b568-63bf1a281b9a\"\n  Description: Pod ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-22T09:53:49Z\n  Ended:       2026-08-22T10:12:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3442688-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3442688/providers/microsoft.monitor/accounts/hcps-j3442688\", namespace=\"ocm-arohcpci01-2sbhtcj4djmllishlrj1ubeicnu0g5pf-sre-hcp-cluster\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"73b11fbd-18cc-490d-bc49-f492731b9079\"\n  Description: Pod ocm-arohcpci01-2sbhtcj4djmllishlrj1ubeicnu0g5pf-sre-hcp-cluster/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-22T09:54:46Z\n  Ended:       2026-08-22T10:12:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3442688-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3442688/providers/microsoft.monitor/accounts/hcps-j3442688\", namespace=\"ocm-arohcpci01-2sbhtcj4djmllishlrj1ubeicnu0g5pf-sre-hcp-cluster\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"73b11fbd-18cc-490d-bc49-f492731b9079\"\n  Description: Pod ocm-arohcpci01-2sbhtcj4djmllishlrj1ubeicnu0g5pf-sre-hcp-cluster/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092146717536292864","occurred_at":"2026-08-25T07:07:02Z","signature_id":"592b27ee04df02dab6c1e953326523943126040c3e195f04af0ccc2cb082c127","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"9d5e8ec654bc2ebd399a25878bada85329c8658827f68d63b0042fb7498d8047","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091085161763442688","occurred_at":"2026-08-22T08:48:46Z","signature_id":"7c60f219c222a62d33851943284ebbb72e284ca27ae434afbd2eb93ae7311af5","pr_number":6640,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090960143322714112","occurred_at":"2026-08-22T00:32:00Z","signature_id":"67afc4691e036888d68d3d1a097dff27f3dee600ac245480c61c302a28649421","pr_number":6639,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090465920358027264","occurred_at":"2026-08-20T15:48:08Z","signature_id":"2e5d452b37114ba6800554f43052dc62e06de28d819190587202da2dd8d76bd2","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090405439429677056","occurred_at":"2026-08-20T11:47:38Z","signature_id":"ff1b491f2f616f8c76c519fe7e81f5344a08bdc867b275f3ca98624a57a64102","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090191599463043072","occurred_at":"2026-08-19T21:38:05Z","signature_id":"beaf2c9e6ccf7a0afb381b01f5e95b286fbc41966ac0bd4afa4af24042def984","pr_number":6608,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090133175878553600","occurred_at":"2026-08-19T17:45:52Z","signature_id":"0a1f68d8ba1568ae18db579b0b5ca5f8a090856f14ce7ebf1a338ca265ecc142","pr_number":6601,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.2618296529968454,"anchor_occurrences":8,"after_last_push_count":5,"trend_counts":[1,3,2,2,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":51,"prior_last_seen_at":"2026-08-21T05:44:12Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":8}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T14:40:10Z\n  Ended:       2026-08-24T14:49:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6438784-svc\", component=\"backend\", name=\"csstatedump\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-21T07:30:21Z\n  Ended:       2026-08-21T07:38:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4553984-svc\", component=\"backend\", name=\"csstatedump\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-21T07:31:20Z\n  Ended:       2026-08-21T07:38:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4553984-svc\", component=\"backend\", name=\"datadump\", severity=\"warning\"\n  Description: Backend controller workqueue datadump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 3 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-21T05:41:29Z\n  Ended:       2026-08-21T05:56:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8412928-svc\", component=\"backend\", name=\"createclusterscopedreaddesires\", severity=\"warning\"\n  Description: Backend controller workqueue createclusterscopedreaddesires has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-21T05:41:29Z\n  Ended:       2026-08-21T05:56:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8412928-svc\", component=\"backend\", name=\"billingdump\", severity=\"warning\"\n  Description: Backend controller workqueue billingdump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-21T05:41:29Z\n  Ended:       2026-08-21T05:56:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8412928-svc\", component=\"backend\", name=\"datadump\", severity=\"warning\"\n  Description: Backend controller workqueue datadump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879927736438784","occurred_at":"2026-08-24T13:26:52Z","signature_id":"f42ba6cef1b20012612134284fc91db6cdee01f2a97291f342457df1df0028e6","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"a776dc9dd0c07bc72a2a24fcac7b5cf950c5185ee842e2520407e59933eb5fe6","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"6df0be81a04ab8fcbefe9cec40b0a31944793bf2405e2b497fc22dcf57c31988","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6622/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090464934205853696","occurred_at":"2026-08-20T15:44:12Z","signature_id":"0650add4c8ee71c20554737cc9c8a8701b1ecdb4bb6417db99f807fb7975fcec","pr_number":6622,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6616/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090329417594179584","occurred_at":"2026-08-20T06:45:44Z","signature_id":"2045d121c82435792aa22041b6c6dbf2f0fd58077598faaae5237ca9ed5f314e","pr_number":6616,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090198433062195200","occurred_at":"2026-08-19T22:05:11Z","signature_id":"38a5c8e8a2f89b4898206fcbe6a9636f06fc0c2617db9418c6b5e3fb64c7bbfe","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090164198184062976","occurred_at":"2026-08-19T19:49:12Z","signature_id":"16ac7e70f1586d9f026debf16669d9c9370e34cf4b8d7dd6a2443ad454bba160","pr_number":6588,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090068841211105280","occurred_at":"2026-08-19T13:30:10Z","signature_id":"3a423c8780dda4ffb00c339498942ffdd379852403a9bf6d8301e5743d8d3aa2","pr_number":6583,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6bfd7287cb264b36e48f23ab625702748569a1f6c62cf922cf2f1ddfc6b7af87","failure_pattern":"alert [svc] UJClusterProvisionErrors6h30m fired","search_query":"alert [svc] UJClusterProvisionErrors6h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors6h30m does not fire","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.2618296529968454,"anchor_occurrences":8,"after_last_push_count":2,"trend_counts":[0,0,0,1,0,0,0,7],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":27,"prior_last_seen_at":"2026-08-21T01:50:52Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors6h30m does not fire","occurrences":8}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T17:11:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"UJClusterProvisionErrors6h30m\", cluster=\"ci01-j0672512-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0672512/providers/microsoft.monitor/accounts/services-j0672512\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n  Description: More than 30% of cluster create (install) operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T15:14:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"UJClusterProvisionErrors6h30m\", cluster=\"ci01-j9549056-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9549056/providers/microsoft.monitor/accounts/services-j9549056\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n  Description: More than 30% of cluster create (install) operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T12:06:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"UJClusterProvisionErrors6h30m\", cluster=\"ci01-j9005312-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9005312/providers/microsoft.monitor/accounts/services-j9005312\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n  Description: More than 30% of cluster create (install) operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"23aa9d7dee7c9e529060801e9167a47fb7a497985cb1d61e9db28b65884ea401","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"38c81dc69b06278ed233e0939f35e8c71cf8eff2f9cb495db24d74f240d5e1a1","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"af8dcfdf2cbedd9372f04cbd90abd56a127bdc4079bafb1ef1913edb6177347d","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"8e7a90f4497ef1b51e90f2043b6517b8c68d8bfd66277c1d9bf290617a22ad90","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"77f2e0ebb4f832bfccfdc5298c2a752e61abc579b540c5ba9eb248bda7ab6b07","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"c4c245d9532eb880b4ccd3795a4e4bdc7b51cc2471678008cfc9c24b079969f8","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"27c12cddda1a8af9ca8f43e94a34fd689832822c5edc234c8b00528b723b3ae1","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"34c0cccba7f7d6217621e9007ff28fd8c26e2b4db86f8de6918886788462ec89","pr_number":6617,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1e1c28b43b9490600a25203b94e222bbccfa5a8eac1aa50a00348fd1b2cc72d0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":8,"runs_affected":7,"failed_runs":7,"run_impact_percent":1.1041009463722398,"anchor_occurrences":8,"after_last_push_count":0,"also_in":["STG"],"trend_counts":[0,0,0,0,1,0,1,6],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-22T18:55:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cce280\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rg4h7xfrjp8d\" to finish creating: GET https://rp.j4940032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e2c6a53-377f-4d0e-9bd7-168eb63a2802\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e2c6a53-377f-4d0e-9bd7-168eb63a2802\",\n      \"name\": \"9e2c6a53-377f-4d0e-9bd7-168eb63a2802\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T15:24:41.72360651Z\",\n      \"endTime\": \"2026-08-25T15:43:49.738296088Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-version-operator, cluster-storage-operator, ingress-operator, hosted-cluster-config-operator, packageserver, cluster-node-tuning-operator, olm-operator, redhat-operators-catalog, community-operators-catalog, dns-operator, catalog-operator, certified-operators-catalog, cluster-policy-controller, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cce280\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rg4h7xfrjp8d\" to finish creating: GET https://rp.j4940032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e2c6a53-377f-4d0e-9bd7-168eb63a2802\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e2c6a53-377f-4d0e-9bd7-168eb63a2802\",\n      \"name\": \"9e2c6a53-377f-4d0e-9bd7-168eb63a2802\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T15:24:41.72360651Z\",\n      \"endTime\": \"2026-08-25T15:43:49.738296088Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-version-operator, cluster-storage-operator, ingress-operator, hosted-cluster-config-operator, packageserver, cluster-node-tuning-operator, olm-operator, redhat-operators-catalog, community-operators-catalog, dns-operator, catalog-operator, certified-operators-catalog, cluster-policy-controller, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-pgpvxwpm6mjj/cluster-candidate-4-22-bnptgd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001084220\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-bnptgd: failed waiting for cluster=\"cluster-candidate-4-22-bnptgd\" in resourcegroup=\"rg-candidate-4-22-pgpvxwpm6mjj\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b420dfc-8b86-46f6-8ff3-d7254009e5e3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b420dfc-8b86-46f6-8ff3-d7254009e5e3\",\n      \"name\": \"9b420dfc-8b86-46f6-8ff3-d7254009e5e3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:05.515128304Z\",\n      \"endTime\": \"2026-08-25T11:24:07.660901963Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, community-operators-catalog, cluster-node-tuning-operator, catalog-operator, certified-operators-catalog, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-image-registry-operator, redhat-operators-catalog, cluster-version-operator, cluster-network-operator, dns-operator, ingress-operator, packageserver, openshift-controller-manager, cluster-storage-operator, olm-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-pgpvxwpm6mjj/cluster-candidate-4-22-bnptgd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001084220\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-bnptgd: failed waiting for cluster=\"cluster-candidate-4-22-bnptgd\" in resourcegroup=\"rg-candidate-4-22-pgpvxwpm6mjj\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b420dfc-8b86-46f6-8ff3-d7254009e5e3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b420dfc-8b86-46f6-8ff3-d7254009e5e3\",\n      \"name\": \"9b420dfc-8b86-46f6-8ff3-d7254009e5e3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:05.515128304Z\",\n      \"endTime\": \"2026-08-25T11:24:07.660901963Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, community-operators-catalog, cluster-node-tuning-operator, catalog-operator, certified-operators-catalog, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-image-registry-operator, redhat-operators-catalog, cluster-version-operator, cluster-network-operator, dns-operator, ingress-operator, packageserver, openshift-controller-manager, cluster-storage-operator, olm-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f761c0\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-tsn8fjg7z8v8\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c60a9b4-98e8-437a-b0d1-340131fafe1e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c60a9b4-98e8-437a-b0d1-340131fafe1e\",\n      \"name\": \"7c60a9b4-98e8-437a-b0d1-340131fafe1e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:30.548851412Z\",\n      \"endTime\": \"2026-08-25T11:21:35.946063021Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, catalog-operator, cluster-version-operator, openshift-route-controller-manager, olm-operator, packageserver, certified-operators-catalog, cluster-node-tuning-operator, dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, redhat-marketplace-catalog, cluster-storage-operator, ingress-operator, redhat-operators-catalog, openshift-controller-manager, community-operators-catalog, hosted-cluster-config-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f761c0\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-tsn8fjg7z8v8\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c60a9b4-98e8-437a-b0d1-340131fafe1e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c60a9b4-98e8-437a-b0d1-340131fafe1e\",\n      \"name\": \"7c60a9b4-98e8-437a-b0d1-340131fafe1e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:30.548851412Z\",\n      \"endTime\": \"2026-08-25T11:21:35.946063021Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, catalog-operator, cluster-version-operator, openshift-route-controller-manager, olm-operator, packageserver, certified-operators-catalog, cluster-node-tuning-operator, dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, redhat-marketplace-catalog, cluster-storage-operator, ingress-operator, redhat-operators-catalog, openshift-controller-manager, community-operators-catalog, hosted-cluster-config-operator, cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092257519584940032","occurred_at":"2026-08-25T14:27:19Z","signature_id":"1420b0f2188f90f92326018bb88dfc5b387aae62ec7dcbd9fa65850c57c8d47e","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"1f069fa0311578db887619d6e5fa3d28f357906bf03f02fdb94770dedff8043b","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"7e3686375be61c20eb803573e751563f0941ff5159af2cf0a463b90d62b70143","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"b6335e195fbae35825fcde7af066eb7de6c819b9fcc823ca2c6c99c2440ef185","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"1cb8534c4d4dcf75a5e26bdcd0fd6bf39f4b3093ae4d3e32c0a5e2dd8fbdb573","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"946e42b1ec8ff2dfdb875eabb2e23ed3cccd0fb9157427c09f12a2be51c45b2b","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"6b56df09ac5297e89b7853c7486e42c460930ae4b037271fddfef92b6ce0846a","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091237883951190016","occurred_at":"2026-08-22T18:55:32Z","signature_id":"d9db2ecb155b483c3a1b51ce5eaa636dea02ee486d2e3c06c7f286e33f5c9a7c","pr_number":6557,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e42c684cbff8153f648b830fb145663fcbe7dcbcc9111d106802aa7f03e55e71","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":8,"runs_affected":7,"failed_runs":7,"run_impact_percent":1.1041009463722398,"anchor_occurrences":8,"after_last_push_count":0,"also_in":["STG"],"trend_counts":[1,0,1,0,0,0,1,6],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":9,"prior_last_seen_at":"2026-08-20T17:48:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ee8180\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rnmt78kw8wzg\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae74a4ae-565f-4545-9b82-af68f6c7938b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae74a4ae-565f-4545-9b82-af68f6c7938b\",\n      \"name\": \"ae74a4ae-565f-4545-9b82-af68f6c7938b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:19:28.849719343Z\",\n      \"endTime\": \"2026-08-25T19:38:30.742891601Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, openshift-controller-manager, azure-cloud-controller-manager, cluster-image-registry-operator, olm-operator, cluster-version-operator, ignition-server, redhat-operators-catalog, community-operators-catalog, control-plane-pki-operator, cluster-policy-controller, cluster-storage-operator, openshift-apiserver, hosted-cluster-config-operator, cluster-network-operator, catalog-operator, cluster-node-tuning-operator, ignition-server-proxy, ingress-operator, olm-collect-profiles, certified-operators-catalog, machine-approver, kube-controller-manager, dns-operator, konnectivity-agent, redhat-marketplace-catalog, cluster-autoscaler, openshift-route-controller-manager, csi-snapshot-controller-operator, packageserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ee8180\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rnmt78kw8wzg\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae74a4ae-565f-4545-9b82-af68f6c7938b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ae74a4ae-565f-4545-9b82-af68f6c7938b\",\n      \"name\": \"ae74a4ae-565f-4545-9b82-af68f6c7938b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:19:28.849719343Z\",\n      \"endTime\": \"2026-08-25T19:38:30.742891601Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, openshift-controller-manager, azure-cloud-controller-manager, cluster-image-registry-operator, olm-operator, cluster-version-operator, ignition-server, redhat-operators-catalog, community-operators-catalog, control-plane-pki-operator, cluster-policy-controller, cluster-storage-operator, openshift-apiserver, hosted-cluster-config-operator, cluster-network-operator, catalog-operator, cluster-node-tuning-operator, ignition-server-proxy, ingress-operator, olm-collect-profiles, certified-operators-catalog, machine-approver, kube-controller-manager, dns-operator, konnectivity-agent, redhat-marketplace-catalog, cluster-autoscaler, openshift-route-controller-manager, csi-snapshot-controller-operator, packageserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000942ca0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kzttj9k8trpk\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/848dc16e-8d2f-492e-ad78-0e05ac46ed06\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/848dc16e-8d2f-492e-ad78-0e05ac46ed06\",\n      \"name\": \"848dc16e-8d2f-492e-ad78-0e05ac46ed06\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:22:26.59789954Z\",\n      \"endTime\": \"2026-08-25T16:41:28.524110603Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, ignition-server-proxy, cluster-policy-controller, hosted-cluster-config-operator, olm-collect-profiles, redhat-marketplace-catalog, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, cluster-node-tuning-operator, ingress-operator, packageserver, openshift-apiserver, cluster-image-registry-operator, olm-operator, openshift-controller-manager, catalog-operator, community-operators-catalog, ignition-server, redhat-operators-catalog, konnectivity-agent, kube-scheduler, azure-cloud-controller-manager, certified-operators-catalog, machine-approver, cluster-storage-operator, dns-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000942ca0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kzttj9k8trpk\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/848dc16e-8d2f-492e-ad78-0e05ac46ed06\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/848dc16e-8d2f-492e-ad78-0e05ac46ed06\",\n      \"name\": \"848dc16e-8d2f-492e-ad78-0e05ac46ed06\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:22:26.59789954Z\",\n      \"endTime\": \"2026-08-25T16:41:28.524110603Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, ignition-server-proxy, cluster-policy-controller, hosted-cluster-config-operator, olm-collect-profiles, redhat-marketplace-catalog, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, cluster-node-tuning-operator, ingress-operator, packageserver, openshift-apiserver, cluster-image-registry-operator, olm-operator, openshift-controller-manager, catalog-operator, community-operators-catalog, ignition-server, redhat-operators-catalog, konnectivity-agent, kube-scheduler, azure-cloud-controller-manager, certified-operators-catalog, machine-approver, cluster-storage-operator, dns-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014d4ba0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-djpx8vzhvdqj\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a29c2b48-aa8b-4185-a975-2ba36d80abfa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a29c2b48-aa8b-4185-a975-2ba36d80abfa\",\n      \"name\": \"a29c2b48-aa8b-4185-a975-2ba36d80abfa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:32.971395174Z\",\n      \"endTime\": \"2026-08-25T11:20:37.411972437Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, control-plane-pki-operator, cluster-storage-operator, dns-operator, ingress-operator, ignition-server, olm-collect-profiles, certified-operators-catalog, kube-controller-manager, openshift-apiserver, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, cluster-network-operator, konnectivity-agent, ignition-server-proxy, olm-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, cluster-autoscaler, machine-approver, packageserver, kube-scheduler, azure-cloud-controller-manager, cluster-image-registry-operator, catalog-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014d4ba0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-djpx8vzhvdqj\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a29c2b48-aa8b-4185-a975-2ba36d80abfa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a29c2b48-aa8b-4185-a975-2ba36d80abfa\",\n      \"name\": \"a29c2b48-aa8b-4185-a975-2ba36d80abfa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:32.971395174Z\",\n      \"endTime\": \"2026-08-25T11:20:37.411972437Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, control-plane-pki-operator, cluster-storage-operator, dns-operator, ingress-operator, ignition-server, olm-collect-profiles, certified-operators-catalog, kube-controller-manager, openshift-apiserver, cluster-version-operator, openshift-controller-manager, openshift-route-controller-manager, cluster-network-operator, konnectivity-agent, ignition-server-proxy, olm-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, cluster-autoscaler, machine-approver, packageserver, kube-scheduler, azure-cloud-controller-manager, cluster-image-registry-operator, catalog-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"24ef7c09d5d3ddd04909d1a88daf40487983c7d0988d0358607e410a3fcbd326","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"723d424b638d374276f7f3771ede71c77d1aeb31880363605a2c07414b4ed15d","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"9a6939f0032bf8fdc09bb65ca1155a9ec01d12b02a56acd5a25adfca6beb61f3","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"9c1b8b2a2c406c20c5278d4329b8f1b42ee01e05e65a61c633a54cbe6a5372e5","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"17d508ac7b58e43e34e867e2daa42e985d102b6efa1571d29f852cfe4e849659","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"bac68d05e70928d803a2e96e39a3594853024ebe374dbc961bb7717d91c26078","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"14a501f15a848fe949827032cbe7eab1732753bec28bb5cacc0d3876ab277f28","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"646c9d10959bdcfabf83f428c38022cd3ddc2083f56da46a37cd6ed226488e71","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ea8fad35cd89a01793d212abe110c72e86c225b2404d2a441b9893a047eca4bb","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":8,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.7886435331230284,"anchor_occurrences":8,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2,6],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f86580\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qvdpwsdrhf5d\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/11ae9dbb-7a03-4823-a826-d4717833e908\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/11ae9dbb-7a03-4823-a826-d4717833e908\",\n      \"name\": \"11ae9dbb-7a03-4823-a826-d4717833e908\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:13:50.746895077Z\",\n      \"endTime\": \"2026-08-25T16:32:59.846114727Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f86580\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qvdpwsdrhf5d\" to finish creating: GET https://rp.j3036032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/11ae9dbb-7a03-4823-a826-d4717833e908\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/11ae9dbb-7a03-4823-a826-d4717833e908\",\n      \"name\": \"11ae9dbb-7a03-4823-a826-d4717833e908\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:13:50.746895077Z\",\n      \"endTime\": \"2026-08-25T16:32:59.846114727Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-x6bg4m6clbjq/cluster-candidate-4-22-v8dh2g should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001724ba0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-v8dh2g: failed waiting for cluster=\"cluster-candidate-4-22-v8dh2g\" in resourcegroup=\"rg-candidate-4-22-x6bg4m6clbjq\" to finish creating: GET https://rp.j8970496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8b6be2ed-087d-4652-a133-bf375421a904\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8b6be2ed-087d-4652-a133-bf375421a904\",\n      \"name\": \"8b6be2ed-087d-4652-a133-bf375421a904\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:40:17.286340435Z\",\n      \"endTime\": \"2026-08-25T14:59:19.220463305Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-x6bg4m6clbjq/cluster-candidate-4-22-v8dh2g should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001724ba0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-v8dh2g: failed waiting for cluster=\"cluster-candidate-4-22-v8dh2g\" in resourcegroup=\"rg-candidate-4-22-x6bg4m6clbjq\" to finish creating: GET https://rp.j8970496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8b6be2ed-087d-4652-a133-bf375421a904\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8b6be2ed-087d-4652-a133-bf375421a904\",\n      \"name\": \"8b6be2ed-087d-4652-a133-bf375421a904\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:40:17.286340435Z\",\n      \"endTime\": \"2026-08-25T14:59:19.220463305Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000da43a0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-52lwzkdvrvxq\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1c5237ca-aea6-4af4-af59-1bac03bfc222\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1c5237ca-aea6-4af4-af59-1bac03bfc222\",\n      \"name\": \"1c5237ca-aea6-4af4-af59-1bac03bfc222\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:09.556002909Z\",\n      \"endTime\": \"2026-08-25T14:42:14.793443561Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000da43a0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-52lwzkdvrvxq\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1c5237ca-aea6-4af4-af59-1bac03bfc222\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1c5237ca-aea6-4af4-af59-1bac03bfc222\",\n      \"name\": \"1c5237ca-aea6-4af4-af59-1bac03bfc222\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:09.556002909Z\",\n      \"endTime\": \"2026-08-25T14:42:14.793443561Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269862373036032","occurred_at":"2026-08-25T15:16:22Z","signature_id":"4bc11107f133f9177edbdf6e9f119a737521511f6edc5e691bc1878c9ba1f527","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6677/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092248818148970496","occurred_at":"2026-08-25T13:52:37Z","signature_id":"9de0b947dcfc7f945c30a376f6c6b3a6af42020e652c40b79da8b091bb68bf00","pr_number":6677,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"2f9c966df057075e45f28d9a86360c4da74e7efb2d31b2a981af2a97722900a4","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"dabbb7f9069ce3251099eefa51315c3d0e7f90c58175bd7bad32d9f2f8d81b40","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"e8bbe3fd18272cae3ce8b375d491cf16615e129bb95cc62f5766586a6be8de21","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"9fed5947fac9eb6a4be4cefb16ab0947113474d3eee1eb30b287e0d183316002","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"ab3b51419f8536317834b49d26ead032c3d2df538686a40306860f09479fc7a4","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"fb242b6b09da99dc7298c2563fdd22c18c2c33f65971895aaf866ce3967b98d3","pr_number":6641,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a336a16a5821e0f9bdc957e0ad25795b90745b377edb258591a36a05e1d4d901","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":1.1041009463722398,"anchor_occurrences":7,"after_last_push_count":1,"trend_counts":[0,1,0,0,1,0,0,5],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":10,"prior_last_seen_at":"2026-08-22T00:25:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-fnczl99xh8kl/cluster-candidate-4-21-jscsmk should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00097d140\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-jscsmk: failed waiting for cluster=\"cluster-candidate-4-21-jscsmk\" in resourcegroup=\"rg-candidate-4-21-fnczl99xh8kl\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c896c3fa-e9be-48ff-abf7-d53fe9674288\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c896c3fa-e9be-48ff-abf7-d53fe9674288\",\n      \"name\": \"c896c3fa-e9be-48ff-abf7-d53fe9674288\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:27.440763508Z\",\n      \"endTime\": \"2026-08-25T16:40:28.603771658Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-fnczl99xh8kl/cluster-candidate-4-21-jscsmk should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00097d140\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-jscsmk: failed waiting for cluster=\"cluster-candidate-4-21-jscsmk\" in resourcegroup=\"rg-candidate-4-21-fnczl99xh8kl\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c896c3fa-e9be-48ff-abf7-d53fe9674288\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c896c3fa-e9be-48ff-abf7-d53fe9674288\",\n      \"name\": \"c896c3fa-e9be-48ff-abf7-d53fe9674288\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:27.440763508Z\",\n      \"endTime\": \"2026-08-25T16:40:28.603771658Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e88e20\u003e: \n    failed to create HCP cluster ea-listhfgh7b: failed waiting for cluster=\"ea-listhfgh7b\" in resourcegroup=\"ea-list-6p4sr9k5q7bf\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62436e44-f35d-4583-a471-e3b8c1476144\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62436e44-f35d-4583-a471-e3b8c1476144\",\n      \"name\": \"62436e44-f35d-4583-a471-e3b8c1476144\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:30.392692104Z\",\n      \"endTime\": \"2026-08-25T14:42:33.727154801Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e88e20\u003e: \n    failed to create HCP cluster ea-listhfgh7b: failed waiting for cluster=\"ea-listhfgh7b\" in resourcegroup=\"ea-list-6p4sr9k5q7bf\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62436e44-f35d-4583-a471-e3b8c1476144\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/62436e44-f35d-4583-a471-e3b8c1476144\",\n      \"name\": \"62436e44-f35d-4583-a471-e3b8c1476144\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:30.392692104Z\",\n      \"endTime\": \"2026-08-25T14:42:33.727154801Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b8c1c0\u003e: \n    failed to create HCP cluster ea-listwl9qrq: failed waiting for cluster=\"ea-listwl9qrq\" in resourcegroup=\"ea-list-cs4nbh2vqlnx\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee1124f8-4c79-44f0-8dc1-bd5636fec70e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee1124f8-4c79-44f0-8dc1-bd5636fec70e\",\n      \"name\": \"ee1124f8-4c79-44f0-8dc1-bd5636fec70e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:19.900770335Z\",\n      \"endTime\": \"2026-08-25T11:20:27.863710783Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b8c1c0\u003e: \n    failed to create HCP cluster ea-listwl9qrq: failed waiting for cluster=\"ea-listwl9qrq\" in resourcegroup=\"ea-list-cs4nbh2vqlnx\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee1124f8-4c79-44f0-8dc1-bd5636fec70e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee1124f8-4c79-44f0-8dc1-bd5636fec70e\",\n      \"name\": \"ee1124f8-4c79-44f0-8dc1-bd5636fec70e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:19.900770335Z\",\n      \"endTime\": \"2026-08-25T11:20:27.863710783Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"2501774b6fdfd7c419479e578ae84628be587408415c6fa7b689447828db6eaa","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"de16d3bee3a53f0651e2d553859d03cd36da7aee381249015bf8db424e02b9bb","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"31a5fbbcc1ff5e782a4df57bfa09d31a765c01a8df5db6afadfacac658e620d6","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"c4caf661a44dbe4ab9a819e2a4f4d4a40e4840ddd13380deac01741d243deeb9","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"cc82f7ed850ed911e19f3f8933725fdb7d487b5ae31fdcadcf8ffd1df92eee0f","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090958470403592192","occurred_at":"2026-08-22T00:25:22Z","signature_id":"a58d3dae210b0ca58aefc8f28c941d375561d97193e90e960861be497d6215a9","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"bef5be2331817cad7a640f5b07ec3c12d25ea81d5e5773389458ac86e4d299d5","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"39f402835fcad2b4e1568707b53561f49a8f1538fcb04076b0e82f24e2268915","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":7,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.9463722397476341,"anchor_occurrences":7,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,7],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005e2b40\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-gvqsnvnk8pkx\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60298cc3-cc25-45ff-b759-a7365c10ec56\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60298cc3-cc25-45ff-b759-a7365c10ec56\",\n      \"name\": \"60298cc3-cc25-45ff-b759-a7365c10ec56\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:21.389463971Z\",\n      \"endTime\": \"2026-08-25T19:39:22.423838537Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005e2b40\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-gvqsnvnk8pkx\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60298cc3-cc25-45ff-b759-a7365c10ec56\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/60298cc3-cc25-45ff-b759-a7365c10ec56\",\n      \"name\": \"60298cc3-cc25-45ff-b759-a7365c10ec56\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:21.389463971Z\",\n      \"endTime\": \"2026-08-25T19:39:22.423838537Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0019daa40\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-g5pwkrb85fq6\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a089fdb6-b47d-4203-b58e-db6f9734d598\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a089fdb6-b47d-4203-b58e-db6f9734d598\",\n      \"name\": \"a089fdb6-b47d-4203-b58e-db6f9734d598\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:21:12.44361008Z\",\n      \"endTime\": \"2026-08-25T14:40:13.632066828Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0019daa40\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-g5pwkrb85fq6\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a089fdb6-b47d-4203-b58e-db6f9734d598\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a089fdb6-b47d-4203-b58e-db6f9734d598\",\n      \"name\": \"a089fdb6-b47d-4203-b58e-db6f9734d598\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:21:12.44361008Z\",\n      \"endTime\": \"2026-08-25T14:40:13.632066828Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-5vxfzn with version 4.21.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005eed00\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-5vxfzn: failed waiting for cluster=\"np-version-upgrade-cluster-5vxfzn\" in resourcegroup=\"rg-np-version-upgrade-5vxfzn-7svm57fp7wk7\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1fa03eeb-ac17-4494-af34-d2fa27ce5596\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1fa03eeb-ac17-4494-af34-d2fa27ce5596\",\n      \"name\": \"1fa03eeb-ac17-4494-af34-d2fa27ce5596\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:54.133785755Z\",\n      \"endTime\": \"2026-08-25T11:20:57.201457228Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, olm-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-5vxfzn with version 4.21.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005eed00\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-5vxfzn: failed waiting for cluster=\"np-version-upgrade-cluster-5vxfzn\" in resourcegroup=\"rg-np-version-upgrade-5vxfzn-7svm57fp7wk7\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1fa03eeb-ac17-4494-af34-d2fa27ce5596\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1fa03eeb-ac17-4494-af34-d2fa27ce5596\",\n      \"name\": \"1fa03eeb-ac17-4494-af34-d2fa27ce5596\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:54.133785755Z\",\n      \"endTime\": \"2026-08-25T11:20:57.201457228Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, olm-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"383f94fc81a983b134e7f8347429995df9c16fd5e3b4d726b4e05d385fc27709","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"a9c57aba5f42be110377a5cf9ef9c173f8bee026a837d5411882de02fec7d3cc","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"effe32de00004a904c211837be3300222c5b0de9f1d089f9996c139580d10677","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"aabd7b54cdc1ced5ff8794a6e08fcc1a9d50f3d188a6e28369345aad73a983a2","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"2481cd9dd55cb11b365e311cdeb7a7671bbaebdfc0f19a6ea5043f7d815bc754","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"5d116f9528929d7006556d8f84e31a0f89f4b8bd4db46d055fca3f0978d9a496","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"6e775195179703ce3594578f28d014a111b897b75498606daa63da0196f2a8f5","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":7,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.6309148264984227,"anchor_occurrences":7,"after_last_push_count":3,"also_in":["PROD","STG"],"trend_counts":[0,2,0,1,0,0,0,4],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-21T15:05:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019ac228\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-lhs7stnz7jdx\" to finish deleting: GET https://rp.j0912128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee794509-c973-4d3d-9aac-193a29f17172\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee794509-c973-4d3d-9aac-193a29f17172\",\n      \"name\": \"ee794509-c973-4d3d-9aac-193a29f17172\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:40:54.657536996Z\",\n      \"endTime\": \"2026-08-25T17:04:58.539200774Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdn70pm4stdfpd62ob77dhptdq67h82 still exists (deletion dispatched at 2026-08-25T16:40:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019ac228\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-lhs7stnz7jdx\" to finish deleting: GET https://rp.j0912128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee794509-c973-4d3d-9aac-193a29f17172\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ee794509-c973-4d3d-9aac-193a29f17172\",\n      \"name\": \"ee794509-c973-4d3d-9aac-193a29f17172\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:40:54.657536996Z\",\n      \"endTime\": \"2026-08-25T17:04:58.539200774Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdn70pm4stdfpd62ob77dhptdq67h82 still exists (deletion dispatched at 2026-08-25T16:40:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e40258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-m7ds2xdkbpmr\" to finish deleting: GET https://rp.j0912128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/424a05dd-5f96-45bf-a866-906148cef205\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/424a05dd-5f96-45bf-a866-906148cef205\",\n      \"name\": \"424a05dd-5f96-45bf-a866-906148cef205\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:34:49.049807041Z\",\n      \"endTime\": \"2026-08-25T16:58:58.538368578Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdn6lgco9mj0rdvfe15crftvs8408gm still exists (deletion dispatched at 2026-08-25T16:35:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e40258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-m7ds2xdkbpmr\" to finish deleting: GET https://rp.j0912128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/424a05dd-5f96-45bf-a866-906148cef205\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/424a05dd-5f96-45bf-a866-906148cef205\",\n      \"name\": \"424a05dd-5f96-45bf-a866-906148cef205\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:34:49.049807041Z\",\n      \"endTime\": \"2026-08-25T16:58:58.538368578Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdn6lgco9mj0rdvfe15crftvs8408gm still exists (deletion dispatched at 2026-08-25T16:35:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a3a078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-vwd6cf59jxq2\" to finish deleting: GET https://rp.j0912128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb828c3-349f-4ff7-bb15-1cb33bc05bfa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb828c3-349f-4ff7-bb15-1cb33bc05bfa\",\n      \"name\": \"8cb828c3-349f-4ff7-bb15-1cb33bc05bfa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:34:33.14717886Z\",\n      \"endTime\": \"2026-08-25T16:58:38.634142205Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdn6lk5i59n2vdugq91i3qhl1hovg7u still exists (deletion dispatched at 2026-08-25T16:34:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a3a078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-vwd6cf59jxq2\" to finish deleting: GET https://rp.j0912128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb828c3-349f-4ff7-bb15-1cb33bc05bfa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb828c3-349f-4ff7-bb15-1cb33bc05bfa\",\n      \"name\": \"8cb828c3-349f-4ff7-bb15-1cb33bc05bfa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:34:33.14717886Z\",\n      \"endTime\": \"2026-08-25T16:58:38.634142205Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sdn6lk5i59n2vdugq91i3qhl1hovg7u still exists (deletion dispatched at 2026-08-25T16:34:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"83f994f356f863bb37e48c00b2ef945191b4a6e47aa90ca4f068fad7c6f09ca9","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"87e01991e35060c62797d52e4e26918b818f17bdf40e7387d9f553c0a433cb0e","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"9c29cfd69e7610a41eb87b6317c7a73f3a7a8990c6ce0206c424b69b0ec00888","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"fcc9fdbef9bad418f67e51fbdb9dd4579ca43684aae6fa975e7587a95e591f82","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817534927310848","occurred_at":"2026-08-21T15:05:19Z","signature_id":"a489e8aa78d68850b06367b29a4da923ea7dc01f7a8e25f8f3561383918d490f","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090148755918557184","occurred_at":"2026-08-19T18:47:51Z","signature_id":"55ed54e871e5ba9e79f7a10853e03aea5541e1178fbf8962dee673333f8ced14","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089990952641892352","occurred_at":"2026-08-19T08:20:47Z","signature_id":"8bc6366d372ee7089c2b2207c73b9df938b4edc215900845a3fa3a5deb9f14de","pr_number":6538,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410","failure_pattern":"ERROR CODE: DeploymentStackDeploymentFailed","search_query":"ERROR CODE: DeploymentStackDeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.9463722397476341,"anchor_occurrences":6,"after_last_push_count":0,"trend_counts":[2,1,3,2,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":10,"prior_last_seen_at":"2026-08-21T21:59:03Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","occurrences":6}],"full_error_samples":["time=2026-08-21T22:06:45.798Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n  Kind: ARMStack\\n  Template: templates/monitoring.bicep\\n  Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-08-21T22:07:37.070Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/37a71f39-392c-4236-b7bc-d450dfa4c5b6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/37a71f39-392c-4236-b7bc-d450dfa4c5b6\\\",\\n  \\\"name\\\": \\\"37a71f39-392c-4236-b7bc-d450dfa4c5b6\\\",\\n  \\\"status\\\": \\\"failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n    \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'f47e133e-49d1-4df1-8690-f91ad3063ea3'.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4648448/providers/Microsoft.Resources/deployments/y5flko475afn22dntfyewzfj7gfkbiwdmj7ezjquufwse-260821221zuwz\\\",\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n        \\\"details\\\": [\\n          {\\n            \\\"code\\\": \\\"BadRequest\\\",\\n            \\\"message\\\": \\\"Property name: Rules[3].Expression, Attempted value: null, Error: 1:110: parse error: unknown escape sequence U+002E '.' Activity ID: 579c60d6-5bc9-4a52-b767-c8d018c5f149.\\\"\\n          }\\n        ]\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-21T21:50:06.716Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n  Kind: ARMStack\\n  Template: templates/monitoring.bicep\\n  Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-08-21T21:51:27.408Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/dadf4cf0-aee7-46d3-92d2-f8fc4f1e4cab\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/dadf4cf0-aee7-46d3-92d2-f8fc4f1e4cab\\\",\\n  \\\"name\\\": \\\"dadf4cf0-aee7-46d3-92d2-f8fc4f1e4cab\\\",\\n  \\\"status\\\": \\\"failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n    \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'c8f9f91e-9264-4c83-804f-b9f2716a3570'.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8878848/providers/Microsoft.Resources/deployments/y5flko475afn22dntfyewzfj7gfkbiwdmj7ezjquufwse-26082121eod0o\\\",\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n        \\\"details\\\": [\\n          {\\n            \\\"code\\\": \\\"BadRequest\\\",\\n            \\\"message\\\": \\\"Property name: Rules[3].Expression, Attempted value: null, Error: 1:110: parse error: unknown escape sequence U+002E '.' Activity ID: 64eee31e-d466-4285-abd9-d9333b56d87c.\\\"\\n          }\\n        ]\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-20T03:05:18.803Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n  Kind: ARMStack\\n  Template: templates/monitoring.bicep\\n  Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-08-20T03:06:10.220Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/32d5ddb3-12f2-4f69-9adc-281d5a5aee90\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/32d5ddb3-12f2-4f69-9adc-281d5a5aee90\\\",\\n  \\\"name\\\": \\\"32d5ddb3-12f2-4f69-9adc-281d5a5aee90\\\",\\n  \\\"status\\\": \\\"failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n    \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'ed733f51-4922-4e52-8437-524f6699e679'.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3579264/providers/Microsoft.Resources/deployments/y5flko475afn22dntfyewzfj7gfkbiwdmj7ezjquufwse-260820031kmin\\\",\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n        \\\"details\\\": [\\n          {\\n            \\\"code\\\": \\\"BadRequest\\\",\\n            \\\"message\\\": \\\"Property name: Rules[3].Expression, Attempted value: null, Error: 1:110: parse error: unknown escape sequence U+002E '.' Activity ID: b6b6ddad-1c68-456d-966d-a061c184aaf2.\\\"\\n          }\\n        ]\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090921650764648448","occurred_at":"2026-08-21T21:59:03Z","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912921788878848","occurred_at":"2026-08-21T21:24:22Z","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090266981373579264","occurred_at":"2026-08-20T02:37:38Z","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090256601465229312","occurred_at":"2026-08-20T01:56:23Z","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090233522550738944","occurred_at":"2026-08-20T00:24:41Z","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090165593054384128","occurred_at":"2026-08-19T19:54:45Z","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","pr_number":6614,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bad17b00fd2861c199fb8c7332ad5eb58ff1eb35d64efc7066c892df3da78afa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] \u003cno_message\u003e; [clusterServiceStatus] \u003cno_message\u003e; [descendantResources] \u003cno_message\u003e; [hostedCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.9463722397476341,"anchor_occurrences":6,"after_last_push_count":7,"also_in":["STG"],"trend_counts":[1,1,2,2,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":19,"prior_last_seen_at":"2026-08-21T04:29:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c982a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-gdnhnw\" in resourcegroup=\"rg-hs-pre--hm7prkdxbkw6\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\",\n      \"name\": \"67ed1acc-0077-4b07-848d-484f46170f44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:21:50.42303027Z\",\n      \"endTime\": \"2026-08-24T15:45:59.872804145Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c982a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-gdnhnw\" in resourcegroup=\"rg-hs-pre--hm7prkdxbkw6\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\",\n      \"name\": \"67ed1acc-0077-4b07-848d-484f46170f44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:21:50.42303027Z\",\n      \"endTime\": \"2026-08-24T15:45:59.872804145Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001fe00d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-sbns7nl57fsx\" to finish deleting: GET https://rp.j2704896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02c893ba-d638-4b2d-9ac6-b24d9ffc2c41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02c893ba-d638-4b2d-9ac6-b24d9ffc2c41\",\n      \"name\": \"02c893ba-d638-4b2d-9ac6-b24d9ffc2c41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T05:44:31.451258644Z\",\n      \"endTime\": \"2026-08-21T06:08:32.645558792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001fe00d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-sbns7nl57fsx\" to finish deleting: GET https://rp.j2704896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02c893ba-d638-4b2d-9ac6-b24d9ffc2c41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02c893ba-d638-4b2d-9ac6-b24d9ffc2c41\",\n      \"name\": \"02c893ba-d638-4b2d-9ac6-b24d9ffc2c41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T05:44:31.451258644Z\",\n      \"endTime\": \"2026-08-21T06:08:32.645558792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001524090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-znkgn5hxfbf9\" to finish deleting: GET https://rp.j3708160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fec91-844b-4c62-aa4f-0c1bd98999c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fec91-844b-4c62-aa4f-0c1bd98999c6\",\n      \"name\": \"7c4fec91-844b-4c62-aa4f-0c1bd98999c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:32:47.45529919Z\",\n      \"endTime\": \"2026-08-21T02:56:48.748022108Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001524090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-znkgn5hxfbf9\" to finish deleting: GET https://rp.j3708160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fec91-844b-4c62-aa4f-0c1bd98999c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fec91-844b-4c62-aa4f-0c1bd98999c6\",\n      \"name\": \"7c4fec91-844b-4c62-aa4f-0c1bd98999c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:32:47.45529919Z\",\n      \"endTime\": \"2026-08-21T02:56:48.748022108Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"cf9ddb524cae2c0c6a37a93000fda2cd14fe56bff50daec6b5b3ed77ad21467c","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"bee1e84ff0a64e4ca2e0552272b055ce62b94e39bd608e5a2e2955aefb0ca0e9","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090612430223708160","occurred_at":"2026-08-21T01:30:11Z","signature_id":"bd964ef4ccdbb7f347258f9538766dbd5e26f438ced91675a269619c3c825f06","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514755771961344","occurred_at":"2026-08-20T19:02:12Z","signature_id":"c15560f309da4b6beb1a22ff7a9cc58001804d18d46863bfe0ac76d9d56dac64","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090497381098328064","occurred_at":"2026-08-20T17:53:09Z","signature_id":"e618604f8ab9e6904abacd4ab44cb3e1e0af728093b8df4f6cf4ffee68b2721f","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090111264041734144","occurred_at":"2026-08-19T16:18:52Z","signature_id":"c6d70da266bb361671887556071e72f8a400a1404bcfa0d0aff64c5103e4e217","pr_number":6611,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.9463722397476341,"anchor_occurrences":6,"after_last_push_count":9,"trend_counts":[1,3,0,1,1,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":28,"prior_last_seen_at":"2026-08-22T04:44:09Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":6}],"full_error_samples":["alert fired 6 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:21:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:22:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:22:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:23:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:22:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:23:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 6 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-22T06:11:54Z\n  Ended:       2026-08-22T06:23:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/services-j8029824\", namespace=\"klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-22T06:11:54Z\n  Ended:       2026-08-22T06:17:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/services-j8029824\", namespace=\"klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-22T06:11:54Z\n  Ended:       2026-08-22T06:21:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/services-j8029824\", namespace=\"klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-08-22T06:11:54Z\n  Ended:       2026-08-22T06:23:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/services-j8029824\", namespace=\"klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-08-22T06:11:54Z\n  Ended:       2026-08-22T06:19:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/services-j8029824\", namespace=\"klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-08-22T06:11:54Z\n  Ended:       2026-08-22T06:19:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/services-j8029824\", namespace=\"klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T03:07:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6439552-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6439552/providers/microsoft.monitor/accounts/services-j6439552\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-6t89j\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T03:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6439552-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6439552/providers/microsoft.monitor/accounts/services-j6439552\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-6t89j\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"a6a83541f08b4359551ade88cceb74431bde5d979f13cb6852b8c8bc1013fd98","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"317b8c5665259b143d908887f090198d5122fed07b3f6eabfcb499b51735af2d","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"263f0fec58eb18ce8eb5b06529e96eb0994337f8a7d3cdff35b43d545a99a4cd","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090148755918557184","occurred_at":"2026-08-19T18:47:51Z","signature_id":"57fd6a38bf423a3d6bdded79b3ceae9e1993465036803049a90f5483d3e998d2","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"a403c540378f317853254227103a4fcdfcefbdd1200a6e0613ee5843ecd53feb","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089990952641892352","occurred_at":"2026-08-19T08:20:47Z","signature_id":"dbdb72f4c8d3baffa07ee83281d500e2b52443ca8d4794bedc1eed79717204c6","pr_number":6538,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"03e76cdaabc445ed66f9db895033ab5c05ca755861d993eb55fd2ea99afdd76d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":6,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.6309148264984227,"anchor_occurrences":6,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,6],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000af62e0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5j68lm294d7v\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4433c13-3a16-45ba-88e7-ec043217acd4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4433c13-3a16-45ba-88e7-ec043217acd4\",\n      \"name\": \"e4433c13-3a16-45ba-88e7-ec043217acd4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:40.830511566Z\",\n      \"endTime\": \"2026-08-25T11:20:48.170023371Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000af62e0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5j68lm294d7v\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4433c13-3a16-45ba-88e7-ec043217acd4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4433c13-3a16-45ba-88e7-ec043217acd4\",\n      \"name\": \"e4433c13-3a16-45ba-88e7-ec043217acd4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:40.830511566Z\",\n      \"endTime\": \"2026-08-25T11:20:48.170023371Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-bp8trx\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d85000\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-bp8trx: failed waiting for cluster=\"cluster-zstream-5-0-bp8trx\" in resourcegroup=\"rg-zstream-upgrade-5-0-brkg7nwpr7jw\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3b253226-385a-4503-a738-5e57c00054ba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3b253226-385a-4503-a738-5e57c00054ba\",\n      \"name\": \"3b253226-385a-4503-a738-5e57c00054ba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:19.290629965Z\",\n      \"endTime\": \"2026-08-25T11:20:27.735529124Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-bp8trx\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d85000\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-bp8trx: failed waiting for cluster=\"cluster-zstream-5-0-bp8trx\" in resourcegroup=\"rg-zstream-upgrade-5-0-brkg7nwpr7jw\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3b253226-385a-4503-a738-5e57c00054ba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3b253226-385a-4503-a738-5e57c00054ba\",\n      \"name\": \"3b253226-385a-4503-a738-5e57c00054ba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:19.290629965Z\",\n      \"endTime\": \"2026-08-25T11:20:27.735529124Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012005e0\u003e: \n    failed to create HCP cluster np-downgrade-sxtp4r: failed waiting for cluster=\"np-downgrade-sxtp4r\" in resourcegroup=\"rg-np-downgrade-sxtp4r-fhklq2vsgmt4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34134851-0102-4d35-97b3-944ff4b7f92e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34134851-0102-4d35-97b3-944ff4b7f92e\",\n      \"name\": \"34134851-0102-4d35-97b3-944ff4b7f92e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:47.779072949Z\",\n      \"endTime\": \"2026-08-25T11:23:47.559079763Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012005e0\u003e: \n    failed to create HCP cluster np-downgrade-sxtp4r: failed waiting for cluster=\"np-downgrade-sxtp4r\" in resourcegroup=\"rg-np-downgrade-sxtp4r-fhklq2vsgmt4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34134851-0102-4d35-97b3-944ff4b7f92e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34134851-0102-4d35-97b3-944ff4b7f92e\",\n      \"name\": \"34134851-0102-4d35-97b3-944ff4b7f92e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:47.779072949Z\",\n      \"endTime\": \"2026-08-25T11:23:47.559079763Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: ingress-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"2634ba4314b4b33fc2951e3d9e981c753c0c9fdc75af1b8135f694c4402837b6","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"6586291399f5b3fd7dbb07efd25721a7c2283129efed74cec97d843274ba632c","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"fe41252ce6ed75f9abb776a92d3943f65f76fc05052906f4fda9f43fe1b11ce7","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"9228c5a0348f43b2f29683a574e74aac88f34daeeda42ce48207d937f6f8a3c7","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"c3a84dd7ddae67604fc665cb0bbd0f847f461d2effc89e6c2b9a218c130289a2","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"edd0549c792e30ee83baa44f1b97d25996781b695fc96efc50800b1d38ed59e2","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":6,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":6,"after_last_push_count":9,"trend_counts":[0,0,0,0,0,0,0,6],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-14T20:39:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:137]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-wtgj75\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00152e100\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-22-wtgj75-5swmlf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-p5qqvrtstbvw\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-22-p5qqvrtstbvw/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-22-wtgj75-5swmlf/operationStatuses/08584139797161101429\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-vcuqi2ogq2kwk' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:137]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-wtgj75\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00152e100\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-22-wtgj75-5swmlf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-p5qqvrtstbvw\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-22-p5qqvrtstbvw/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-22-wtgj75-5swmlf/operationStatuses/08584139797161101429\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-vcuqi2ogq2kwk' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000839700\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-np-labels-taints-qdpn79\" in resourcegroup=\"rg-np-labels-taints-5t4klzqbfs75\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-5t4klzqbfs75/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-qdpn79/operationStatuses/08584139797158851285\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-eajlprfd56pyu' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000839700\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-np-labels-taints-qdpn79\" in resourcegroup=\"rg-np-labels-taints-5t4klzqbfs75\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-5t4klzqbfs75/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-qdpn79/operationStatuses/08584139797158851285\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-eajlprfd56pyu' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00184e0e0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-listd9kcp5-zjrzlw\" in resourcegroup=\"ea-list-db5549k8k7r7\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/ea-list-db5549k8k7r7/providers/Microsoft.Resources/deployments/customer-infra-ea-listd9kcp5-zjrzlw/operationStatuses/08584139797158020904\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-yygnj2pbjetam' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00184e0e0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-listd9kcp5-zjrzlw\" in resourcegroup=\"ea-list-db5549k8k7r7\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/ea-list-db5549k8k7r7/providers/Microsoft.Resources/deployments/customer-infra-ea-listd9kcp5-zjrzlw/operationStatuses/08584139797158020904\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-yygnj2pbjetam' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"5c83e4b12f76857c14b38ba7e2400380e5142f913e512f879520f40194148815","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"6798ec01d31b1a13a551f0d58e2e35ad1b3449b1679cb8d8c473dee35aed6098","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"af625ee5896ee6ce49c9e654442a846928a680f63a96ce2b39b4c1040f2836c7","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"b2816cbf83c46fd984a6ad136b3252029110012c9e6f9c0547ab7a88bcac4f9a","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"e607efd27bdf5f0f02b0b2627c8fcd0b747075950cb39ab6c422f0ffaddbcc9f","pr_number":6666,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092074867384913920","occurred_at":"2026-08-25T02:21:30Z","signature_id":"eef4016eff4f328bc4e6b8d0c0a4efcaa132ceba7408e01c03a4f07e211176c2","pr_number":6666,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a89f0d2edcd262580a6fd914503ced16ad52ae763f2c2255b4f5af55d375ecd3","failure_pattern":"alert [svc] KubeContainerWaiting fired","search_query":"alert [svc] KubeContainerWaiting fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerWaiting does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.7886435331230284,"anchor_occurrences":5,"after_last_push_count":1,"trend_counts":[0,2,1,1,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":9,"prior_last_seen_at":"2026-08-21T03:34:17Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerWaiting does not fire","occurrences":5}],"full_error_samples":["alert fired 8 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T12:22:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"liveness-probe\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-2k5cw\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-2k5cw in namespace kube-system on container liveness-probe has been in waiting state for longer than 1 hour.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T12:22:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"trivy-scanner\", namespace=\"kube-system\", pod=\"eraser-aks-system-30244119-vmss000000-7fwv2\", severity=\"warning\"\n  Description: pod/eraser-aks-system-30244119-vmss000000-7fwv2 in namespace kube-system on container trivy-scanner has been in waiting state for longer than 1 hour.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T12:22:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", namespace=\"kube-system\", pod=\"ama-metrics-node-rvqrk\", severity=\"warning\"\n  Description: pod/ama-metrics-node-rvqrk in namespace kube-system on container prometheus-collector has been in waiting state for longer than 1 hour.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T12:22:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"secrets-store\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-2k5cw\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-2k5cw in namespace kube-system on container secrets-store has been in waiting state for longer than 1 hour.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T12:22:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"addon-token-adapter\", namespace=\"kube-system\", pod=\"ama-metrics-node-rvqrk\", severity=\"warning\"\n  Description: pod/ama-metrics-node-rvqrk in namespace kube-system on container addon-token-adapter has been in waiting state for longer than 1 hour.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T12:22:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"collector\", namespace=\"kube-system\", pod=\"eraser-aks-system-30244119-vmss000000-7fwv2\", severity=\"warning\"\n  Description: pod/eraser-aks-system-30244119-vmss000000-7fwv2 in namespace kube-system on container collector has been in waiting state for longer than 1 hour.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T12:22:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"remover\", namespace=\"kube-system\", pod=\"eraser-aks-system-30244119-vmss000000-7fwv2\", severity=\"warning\"\n  Description: pod/eraser-aks-system-30244119-vmss000000-7fwv2 in namespace kube-system on container remover has been in waiting state for longer than 1 hour.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T12:22:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1136640-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"node-driver-registrar\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-2k5cw\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-2k5cw in namespace kube-system on container node-driver-registrar has been in waiting state for longer than 1 hour.","alert fired 8 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T05:09:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j4800000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"collector\", namespace=\"kube-system\", pod=\"eraser-aks-system-31747703-vmss000000-rwxmt\", severity=\"warning\"\n  Description: pod/eraser-aks-system-31747703-vmss000000-rwxmt in namespace kube-system on container collector has been in waiting state for longer than 1 hour.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T05:09:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j4800000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"addon-token-adapter\", namespace=\"kube-system\", pod=\"ama-metrics-node-fnzb8\", severity=\"warning\"\n  Description: pod/ama-metrics-node-fnzb8 in namespace kube-system on container addon-token-adapter has been in waiting state for longer than 1 hour.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T05:09:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j4800000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", namespace=\"kube-system\", pod=\"ama-metrics-node-fnzb8\", severity=\"warning\"\n  Description: pod/ama-metrics-node-fnzb8 in namespace kube-system on container prometheus-collector has been in waiting state for longer than 1 hour.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T05:09:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j4800000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"node-driver-registrar\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-w8ttz\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-w8ttz in namespace kube-system on container node-driver-registrar has been in waiting state for longer than 1 hour.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T05:09:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j4800000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"secrets-store\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-w8ttz\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-w8ttz in namespace kube-system on container secrets-store has been in waiting state for longer than 1 hour.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T05:09:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j4800000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"remover\", namespace=\"kube-system\", pod=\"eraser-aks-system-31747703-vmss000000-rwxmt\", severity=\"warning\"\n  Description: pod/eraser-aks-system-31747703-vmss000000-rwxmt in namespace kube-system on container remover has been in waiting state for longer than 1 hour.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T05:09:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j4800000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"trivy-scanner\", namespace=\"kube-system\", pod=\"eraser-aks-system-31747703-vmss000000-rwxmt\", severity=\"warning\"\n  Description: pod/eraser-aks-system-31747703-vmss000000-rwxmt in namespace kube-system on container trivy-scanner has been in waiting state for longer than 1 hour.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T05:09:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j4800000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"liveness-probe\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-w8ttz\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-w8ttz in namespace kube-system on container liveness-probe has been in waiting state for longer than 1 hour.","alert fired 8 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T14:39:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j2425216-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"node-driver-registrar\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-724fc\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-724fc in namespace kube-system on container node-driver-registrar has been in waiting state for longer than 1 hour.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T14:39:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j2425216-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"trivy-scanner\", namespace=\"kube-system\", pod=\"eraser-aks-system-12106338-vmss000000-vcv9l\", severity=\"warning\"\n  Description: pod/eraser-aks-system-12106338-vmss000000-vcv9l in namespace kube-system on container trivy-scanner has been in waiting state for longer than 1 hour.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T14:39:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j2425216-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", namespace=\"kube-system\", pod=\"ama-metrics-node-7nn7h\", severity=\"warning\"\n  Description: pod/ama-metrics-node-7nn7h in namespace kube-system on container prometheus-collector has been in waiting state for longer than 1 hour.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T14:39:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j2425216-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"secrets-store\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-724fc\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-724fc in namespace kube-system on container secrets-store has been in waiting state for longer than 1 hour.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T14:40:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j2425216-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"liveness-probe\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-724fc\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-724fc in namespace kube-system on container liveness-probe has been in waiting state for longer than 1 hour.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T14:40:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j2425216-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"addon-token-adapter\", namespace=\"kube-system\", pod=\"ama-metrics-node-7nn7h\", severity=\"warning\"\n  Description: pod/ama-metrics-node-7nn7h in namespace kube-system on container addon-token-adapter has been in waiting state for longer than 1 hour.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T14:40:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j2425216-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"remover\", namespace=\"kube-system\", pod=\"eraser-aks-system-12106338-vmss000000-vcv9l\", severity=\"warning\"\n  Description: pod/eraser-aks-system-12106338-vmss000000-vcv9l in namespace kube-system on container remover has been in waiting state for longer than 1 hour.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T14:40:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j2425216-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"collector\", namespace=\"kube-system\", pod=\"eraser-aks-system-12106338-vmss000000-vcv9l\", severity=\"warning\"\n  Description: pod/eraser-aks-system-12106338-vmss000000-vcv9l in namespace kube-system on container collector has been in waiting state for longer than 1 hour."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092198034401136640","occurred_at":"2026-08-25T10:30:42Z","signature_id":"41f7f74865411c0e7be6f2be5d79bdefecee7e90cea861519501a2ae2711a449","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090643626764800000","occurred_at":"2026-08-21T03:34:17Z","signature_id":"48b8e2cb57114629b4e6133fb41aeceaefffcf04b5ff1fc459dd655e0252babe","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090419688172425216","occurred_at":"2026-08-20T12:44:10Z","signature_id":"c3f4f642d5fe7b8cbf9f6c9b9e831f3a5a1bb8747828d07eeed3169305d122a4","pr_number":6617,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090180258316685312","occurred_at":"2026-08-19T20:53:01Z","signature_id":"8746d886253ada4663f87ba3097e3a4e02632ca623c664c63a686376132b6822","pr_number":6614,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090031884472946688","occurred_at":"2026-08-19T11:03:26Z","signature_id":"616e2d13a8d730313eb967f4b30c58b8a702753aad7be6756af4bbe5eefbe757","pr_number":6583,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fac505f52469073fa6a64250c64e44813d69018a355acebf1fdde81a505c53f0","failure_pattern":"expected 3 nodes to have original label key1=value1","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.7886435331230284,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[2,1,0,2,0,0,1,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-10","2026-08-17"],"prior_runs_affected":9,"prior_last_seen_at":"2026-08-21T15:05:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092271779060912128","occurred_at":"2026-08-25T15:23:59Z","signature_id":"c01b4fcf6cc6dec59f03daaf85e15cd790cfdad06e2936a348917bd71e8e78e7","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","signature_id":"c01b4fcf6cc6dec59f03daaf85e15cd790cfdad06e2936a348917bd71e8e78e7","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817534927310848","occurred_at":"2026-08-21T15:05:19Z","signature_id":"c01b4fcf6cc6dec59f03daaf85e15cd790cfdad06e2936a348917bd71e8e78e7","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090766839033368576","occurred_at":"2026-08-21T11:43:53Z","signature_id":"c01b4fcf6cc6dec59f03daaf85e15cd790cfdad06e2936a348917bd71e8e78e7","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"c01b4fcf6cc6dec59f03daaf85e15cd790cfdad06e2936a348917bd71e8e78e7","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"db70381daed7f2c0ae7388805235b1692e0241b6bbf9c9fad5adb0318ac4ed06","failure_pattern":"failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"\u003curl\u003e","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.2","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.7886435331230284,"anchor_occurrences":5,"after_last_push_count":1,"trend_counts":[0,0,3,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-20T15:48:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.2","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.129.22.77:37397-\u003e172.30.0.10:53: i/o timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.129.22.77:37397-\u003e172.30.0.10:53: i/o timeout","fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.129.200.17:33911-\u003e172.30.0.10:53: i/o timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.129.200.17:33911-\u003e172.30.0.10:53: i/o timeout","fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.131.82.17:38030-\u003e172.30.0.10:53: i/o timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.131.82.17:38030-\u003e172.30.0.10:53: i/o timeout"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260279839625216","occurred_at":"2026-08-25T14:38:17Z","signature_id":"cf8d75a064f159907166271d9eeaa685334da9829b43b0bb83f85eba1df8e3e4","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6669/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092189876085592064","occurred_at":"2026-08-25T09:52:51Z","signature_id":"0fe7f6172a7cc0e6e9770da225a113c760ce09d48125d6f7381c144d3873bcb3","pr_number":6669,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090465920358027264","occurred_at":"2026-08-20T15:48:08Z","signature_id":"f293df76335ff7a793270b509afdcb832220e5fc494a428bb2d05bcb86a7eacd","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090283103464460288","occurred_at":"2026-08-20T03:41:42Z","signature_id":"036ccb61732bbdd95d1fe66f7d3a178bde3724585269d7d722d62ee6e4611402","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248642563674112","occurred_at":"2026-08-20T01:24:45Z","signature_id":"b55dd7b53b01cb139b15a9a7c10b0647e7752eca515812e2b7f5ba7dc4f7c030","pr_number":6615,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","failure_pattern":"private ingress should not be reachable from outside the VNet, but connection succeeded","search_query":"private ingress should not be reachable from outside the VNet, but connection succeeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.7886435331230284,"anchor_occurrences":5,"after_last_push_count":7,"trend_counts":[1,1,1,2,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":14,"prior_last_seen_at":"2026-08-21T21:06:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:186]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred.  Got:\n    \u003cnil\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:186]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred.  Got:\n    \u003cnil\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092071623086575616","occurred_at":"2026-08-25T02:08:38Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090908436404375552","occurred_at":"2026-08-21T21:06:33Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090618825308377088","occurred_at":"2026-08-21T01:55:44Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514791213830144","occurred_at":"2026-08-20T19:02:20Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090126361980047360","occurred_at":"2026-08-19T17:18:51Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":6584,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"da0050d2b241d5b64bca4e3ea0c184bfa4a846f096a3f477d0e09b223d10f5d6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":5,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.6309148264984227,"anchor_occurrences":5,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,4],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-22T00:10:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f0a1a0\u003e: \n    failed to create HCP cluster ea-listmdvtnb: failed waiting for cluster=\"ea-listmdvtnb\" in resourcegroup=\"ea-list-fwskcqw558j7\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09211a76-ef74-4d73-bfcd-1348d7d2680c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09211a76-ef74-4d73-bfcd-1348d7d2680c\",\n      \"name\": \"09211a76-ef74-4d73-bfcd-1348d7d2680c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:23.028754386Z\",\n      \"endTime\": \"2026-08-25T19:39:32.394595575Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f0a1a0\u003e: \n    failed to create HCP cluster ea-listmdvtnb: failed waiting for cluster=\"ea-listmdvtnb\" in resourcegroup=\"ea-list-fwskcqw558j7\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09211a76-ef74-4d73-bfcd-1348d7d2680c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09211a76-ef74-4d73-bfcd-1348d7d2680c\",\n      \"name\": \"09211a76-ef74-4d73-bfcd-1348d7d2680c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:23.028754386Z\",\n      \"endTime\": \"2026-08-25T19:39:32.394595575Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a84540\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-qsmsgbz4r9bl\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb200329-6a1f-4203-89e8-8bd77fdd326b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb200329-6a1f-4203-89e8-8bd77fdd326b\",\n      \"name\": \"eb200329-6a1f-4203-89e8-8bd77fdd326b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:10.235142906Z\",\n      \"endTime\": \"2026-08-25T11:22:17.989968047Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a84540\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-qsmsgbz4r9bl\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb200329-6a1f-4203-89e8-8bd77fdd326b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb200329-6a1f-4203-89e8-8bd77fdd326b\",\n      \"name\": \"eb200329-6a1f-4203-89e8-8bd77fdd326b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:10.235142906Z\",\n      \"endTime\": \"2026-08-25T11:22:17.989968047Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00069e2c0\u003e: \n    failed to create HCP cluster np-dg-minor-zd5jtq: failed waiting for cluster=\"np-dg-minor-zd5jtq\" in resourcegroup=\"rg-np-dg-minor-zd5jtq-kkl7vl8r2p4z\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\",\n      \"name\": \"20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:30.680263097Z\",\n      \"endTime\": \"2026-08-25T11:20:37.924037446Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00069e2c0\u003e: \n    failed to create HCP cluster np-dg-minor-zd5jtq: failed waiting for cluster=\"np-dg-minor-zd5jtq\" in resourcegroup=\"rg-np-dg-minor-zd5jtq-kkl7vl8r2p4z\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\",\n      \"name\": \"20b0f9df-28c2-45ff-b6bf-ca10517cb3f4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:30.680263097Z\",\n      \"endTime\": \"2026-08-25T11:20:37.924037446Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"8ad26a920569e563149923a5bcd439d71e32d4914bb330af2790f80b14322048","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"1b1a509582594e9612da11037861a2814fe682ec6e6e3e82e70696765264d057","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"83e815c22c5353bc66d4aad1ca57dc77c126e34a396f19d17d4961aa8e291e1b","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"79463fb4058b95541bcd55a0c46b6475494bc057241f46f47f685884fbe58103","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090954745601593344","occurred_at":"2026-08-22T00:10:34Z","signature_id":"0f615daad0c6c3c41aba5ffc09456a12dc1b17c512cf6fcba2046e7323cdd028","pr_number":6593,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"22a2ec90b8e4c03b7eb4b7cf7a664ff8afb21270d440786b9593f19fac9b96a1","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":5,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.6309148264984227,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,5],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fc8b80\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-r67n76nc45fp\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f38a6194-1bb0-4fa8-91b4-9d9b52f87625\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f38a6194-1bb0-4fa8-91b4-9d9b52f87625\",\n      \"name\": \"f38a6194-1bb0-4fa8-91b4-9d9b52f87625\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:31.043336048Z\",\n      \"endTime\": \"2026-08-25T19:39:32.419104707Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fc8b80\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-r67n76nc45fp\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f38a6194-1bb0-4fa8-91b4-9d9b52f87625\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f38a6194-1bb0-4fa8-91b4-9d9b52f87625\",\n      \"name\": \"f38a6194-1bb0-4fa8-91b4-9d9b52f87625\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:31.043336048Z\",\n      \"endTime\": \"2026-08-25T19:39:32.419104707Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-m94cqb294gjz/cluster-candidate-5-0-cd7r5j should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dd6160\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-cd7r5j: failed waiting for cluster=\"cluster-candidate-5-0-cd7r5j\" in resourcegroup=\"rg-candidate-5-0-m94cqb294gjz\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f5a5c184-48e5-41b3-82f5-d53463fe130e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f5a5c184-48e5-41b3-82f5-d53463fe130e\",\n      \"name\": \"f5a5c184-48e5-41b3-82f5-d53463fe130e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:22:32.285332628Z\",\n      \"endTime\": \"2026-08-25T19:41:32.407207308Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-m94cqb294gjz/cluster-candidate-5-0-cd7r5j should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dd6160\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-cd7r5j: failed waiting for cluster=\"cluster-candidate-5-0-cd7r5j\" in resourcegroup=\"rg-candidate-5-0-m94cqb294gjz\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f5a5c184-48e5-41b3-82f5-d53463fe130e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f5a5c184-48e5-41b3-82f5-d53463fe130e\",\n      \"name\": \"f5a5c184-48e5-41b3-82f5-d53463fe130e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:22:32.285332628Z\",\n      \"endTime\": \"2026-08-25T19:41:32.407207308Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c409e0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-g7htcsxqgjh8\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dffc5a1b-8a6e-474f-9d2b-542515a5266b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dffc5a1b-8a6e-474f-9d2b-542515a5266b\",\n      \"name\": \"dffc5a1b-8a6e-474f-9d2b-542515a5266b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:51.512570533Z\",\n      \"endTime\": \"2026-08-25T11:23:58.061151727Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c409e0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-g7htcsxqgjh8\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dffc5a1b-8a6e-474f-9d2b-542515a5266b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dffc5a1b-8a6e-474f-9d2b-542515a5266b\",\n      \"name\": \"dffc5a1b-8a6e-474f-9d2b-542515a5266b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:51.512570533Z\",\n      \"endTime\": \"2026-08-25T11:23:58.061151727Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: cluster-network-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"6ae7c340a3a6185150f01fedd02fdfde8bcb2b6e79e7c90419ba5cf8f2b0a0c6","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"874e564fd64d9955e95921340933ec68b79b24e1dc50ad4f794673936e060215","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"0fa98435175007c5311cd1a7a6087ef4be15b52a462c15b286367a764bab1677","pr_number":6670,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"32446f0653d184d03231d5fec01e7294672fb079eed4f9359b944cae9896c2fc","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"c5a29c38a564aa794f98663d402b32a7d56cfc809fa2b5066f123163d7b250c5","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c210f344f8b31315095a1c725d401f06aa7258ee61130005e0b9120ecf2ff4a8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,5],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c72900\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-8h5h4xz7mbfk\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54b206e2-551d-4d64-9bdd-e7dc0862599a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54b206e2-551d-4d64-9bdd-e7dc0862599a\",\n      \"name\": \"54b206e2-551d-4d64-9bdd-e7dc0862599a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:48:46.60892922Z\",\n      \"endTime\": \"2026-08-25T20:07:46.977733317Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c72900\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-8h5h4xz7mbfk\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54b206e2-551d-4d64-9bdd-e7dc0862599a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54b206e2-551d-4d64-9bdd-e7dc0862599a\",\n      \"name\": \"54b206e2-551d-4d64-9bdd-e7dc0862599a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:48:46.60892922Z\",\n      \"endTime\": \"2026-08-25T20:07:46.977733317Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013503c0\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-k2wkzd68x6rw\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d447528e-038e-4cc3-8a9a-4ae63f05a790\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d447528e-038e-4cc3-8a9a-4ae63f05a790\",\n      \"name\": \"d447528e-038e-4cc3-8a9a-4ae63f05a790\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:48:46.9471217Z\",\n      \"endTime\": \"2026-08-25T20:07:47.037810744Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013503c0\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-k2wkzd68x6rw\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d447528e-038e-4cc3-8a9a-4ae63f05a790\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d447528e-038e-4cc3-8a9a-4ae63f05a790\",\n      \"name\": \"d447528e-038e-4cc3-8a9a-4ae63f05a790\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:48:46.9471217Z\",\n      \"endTime\": \"2026-08-25T20:07:47.037810744Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c008e0\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-qmfzw8r72h4d\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b1d79c0f-6079-47fa-84e7-db57d11d1583\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b1d79c0f-6079-47fa-84e7-db57d11d1583\",\n      \"name\": \"b1d79c0f-6079-47fa-84e7-db57d11d1583\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:51:05.668136899Z\",\n      \"endTime\": \"2026-08-25T20:10:09.078300161Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c008e0\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-qmfzw8r72h4d\" to finish creating: GET https://rp.j0886784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b1d79c0f-6079-47fa-84e7-db57d11d1583\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b1d79c0f-6079-47fa-84e7-db57d11d1583\",\n      \"name\": \"b1d79c0f-6079-47fa-84e7-db57d11d1583\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:51:05.668136899Z\",\n      \"endTime\": \"2026-08-25T20:10:09.078300161Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas; [roleAssignments] role assignments not yet confirmed\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"05e9d38ab4a3aaad4a0b5371ed4af41ae1885a11dde78c40b966902d6bf39c3e","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"4ba0a65bf49c66d944c03681270514049220b21efd9126ab47abc5555d712dc1","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"62d830a8043675e8a90f508cb3493ce60a38c6b2ad4c7bb07df6c3de3aa1da2c","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"b13e609f7295dd74da4f9e8e1f1a399934f7c35335c10bd5927666f26ed38ac2","pr_number":6681,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6681/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092325609760886784","occurred_at":"2026-08-25T18:57:53Z","signature_id":"da7749385b9e21b7d26b42f57364958d539d80f06df4643564a35c6623b5bb10","pr_number":6681,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"50e9a505dfd78e0f2ab37ff9c69326a4b56cb21d9921b84b1255256ca501391e","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[8,5,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-19T14:06:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00145c120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-fbtznfmgpfmh\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/823db68c-79dc-4727-9121-95296501f4bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/823db68c-79dc-4727-9121-95296501f4bf\",\n      \"name\": \"823db68c-79dc-4727-9121-95296501f4bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:01:32.394014466Z\",\n      \"endTime\": \"2026-08-19T15:25:40.293432829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9jdesahjtqdjvpc60o8amh8beqdt still exists (deletion dispatched at 2026-08-19T15:01:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 11 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00145c120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-fbtznfmgpfmh\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/823db68c-79dc-4727-9121-95296501f4bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/823db68c-79dc-4727-9121-95296501f4bf\",\n      \"name\": \"823db68c-79dc-4727-9121-95296501f4bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:01:32.394014466Z\",\n      \"endTime\": \"2026-08-19T15:25:40.293432829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9jdesahjtqdjvpc60o8amh8beqdt still exists (deletion dispatched at 2026-08-19T15:01:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 11 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011603c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-d9djl4vwvd9d\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1922b32-d944-4e67-9298-3423ac7d756d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1922b32-d944-4e67-9298-3423ac7d756d\",\n      \"name\": \"a1922b32-d944-4e67-9298-3423ac7d756d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:21:24.495403431Z\",\n      \"endTime\": \"2026-08-19T15:45:30.286773534Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9jldsfibh07akcms9kk0t1s8lje8 still exists (deletion dispatched at 2026-08-19T15:21:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 13 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011603c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-d9djl4vwvd9d\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1922b32-d944-4e67-9298-3423ac7d756d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1922b32-d944-4e67-9298-3423ac7d756d\",\n      \"name\": \"a1922b32-d944-4e67-9298-3423ac7d756d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:21:24.495403431Z\",\n      \"endTime\": \"2026-08-19T15:45:30.286773534Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9jldsfibh07akcms9kk0t1s8lje8 still exists (deletion dispatched at 2026-08-19T15:21:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 13 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011fc078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-6hk7rf89rgds\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52d6a689-7624-4713-8e33-581b49068dde\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52d6a689-7624-4713-8e33-581b49068dde\",\n      \"name\": \"52d6a689-7624-4713-8e33-581b49068dde\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:00:27.665748341Z\",\n      \"endTime\": \"2026-08-19T15:24:30.376504783Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9ia6c2epu24n3nrjgvgsrmalvbl6 still exists (deletion dispatched at 2026-08-19T15:00:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 11 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011fc078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-6hk7rf89rgds\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52d6a689-7624-4713-8e33-581b49068dde\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52d6a689-7624-4713-8e33-581b49068dde\",\n      \"name\": \"52d6a689-7624-4713-8e33-581b49068dde\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:00:27.665748341Z\",\n      \"endTime\": \"2026-08-19T15:24:30.376504783Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9ia6c2epu24n3nrjgvgsrmalvbl6 still exists (deletion dispatched at 2026-08-19T15:00:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 11 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"1a7c8d9caa8688de4a4ef04fd863ee389fb3f378d9c8a57e5ad29e4240c26533","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"7dd2e6e13f18d3730b3936e8c95ad3dab0591a65b4972dd217ed46e1d47252fe","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"859c1c841e7e068411598a3f82f1d73f33865edd7a233108aac62d1ee6597008","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"8cfdc425aef7a589789ed69f171d6c8db0c98e46bb0da973215ed67927b6011a","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"fcdd64be3e6d3a09882e5be2f0fca23526c058bbbcf10d8bc5931108751b3ba4","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c3db9c7d221b5f14f4cb052b432def48559d4c9a3619db1db26cc57a99142ed9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.6309148264984227,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,1,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-21T04:39:49Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b0afc0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gvltc2gr7wfl\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a8b4b1e-639e-425c-8c76-d099661899e0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a8b4b1e-639e-425c-8c76-d099661899e0\",\n      \"name\": \"9a8b4b1e-639e-425c-8c76-d099661899e0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:24:12.827625738Z\",\n      \"endTime\": \"2026-08-25T14:43:14.39497302Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b0afc0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gvltc2gr7wfl\" to finish creating: GET https://rp.j7603200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a8b4b1e-639e-425c-8c76-d099661899e0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a8b4b1e-639e-425c-8c76-d099661899e0\",\n      \"name\": \"9a8b4b1e-639e-425c-8c76-d099661899e0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:24:12.827625738Z\",\n      \"endTime\": \"2026-08-25T14:43:14.39497302Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011e26e0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lf47q9x8cxz7\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812a5a7d-5729-4953-b916-1cbe8aba438f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812a5a7d-5729-4953-b916-1cbe8aba438f\",\n      \"name\": \"812a5a7d-5729-4953-b916-1cbe8aba438f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:22:21.177808774Z\",\n      \"endTime\": \"2026-08-25T14:41:23.66756422Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011e26e0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lf47q9x8cxz7\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812a5a7d-5729-4953-b916-1cbe8aba438f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/812a5a7d-5729-4953-b916-1cbe8aba438f\",\n      \"name\": \"812a5a7d-5729-4953-b916-1cbe8aba438f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:22:21.177808774Z\",\n      \"endTime\": \"2026-08-25T14:41:23.66756422Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--wkl8qfr9l4cs/cluster-hs-pre-jtp7jx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c0960\u003e: \n    failed to create HCP cluster cluster-hs-pre-jtp7jx: failed waiting for cluster=\"cluster-hs-pre-jtp7jx\" in resourcegroup=\"rg-hs-pre--wkl8qfr9l4cs\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\",\n      \"name\": \"df253c01-fd54-4465-9725-47d0942d268e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:18:48.695053949Z\",\n      \"endTime\": \"2026-08-24T15:37:57.993163664Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--wkl8qfr9l4cs/cluster-hs-pre-jtp7jx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c0960\u003e: \n    failed to create HCP cluster cluster-hs-pre-jtp7jx: failed waiting for cluster=\"cluster-hs-pre-jtp7jx\" in resourcegroup=\"rg-hs-pre--wkl8qfr9l4cs\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\",\n      \"name\": \"df253c01-fd54-4465-9725-47d0942d268e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:18:48.695053949Z\",\n      \"endTime\": \"2026-08-24T15:37:57.993163664Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"b217f98ca261620a2196fa549c1345074f0ad31609adefc4d0a95125d87e751a","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"f0717dee0439ce4b9da20978f82a9a18a9e1f1a19a95e531972424323361a844","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"113a28e0e41ddb16e2d1b2e27f386f97c1c16af7bfd0bd1391cd74394fd64a44","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090660118717796352","occurred_at":"2026-08-21T04:39:49Z","signature_id":"c41bef6c51b50ccc02005212254243b6d9e7442f62932fa4787fc2a446e87426","pr_number":6584,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0fa837367648e2c6015295472e0f8df164e966eed931bdff87d5784d41e8c7f8","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.6309148264984227,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[1,0,0,0,0,0,0,4],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-18T14:39:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e26160\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-q672lrjjt7xx\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ba6b631-6240-4773-bfa1-ea8d27930011\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ba6b631-6240-4773-bfa1-ea8d27930011\",\n      \"name\": \"5ba6b631-6240-4773-bfa1-ea8d27930011\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:18:59.654390099Z\",\n      \"endTime\": \"2026-08-25T19:38:02.42081904Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e26160\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-q672lrjjt7xx\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ba6b631-6240-4773-bfa1-ea8d27930011\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ba6b631-6240-4773-bfa1-ea8d27930011\",\n      \"name\": \"5ba6b631-6240-4773-bfa1-ea8d27930011\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:18:59.654390099Z\",\n      \"endTime\": \"2026-08-25T19:38:02.42081904Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-84dql4hbsnbj/cluster-candidate-5-0-2lchcj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aaf8a0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-2lchcj: failed waiting for cluster=\"cluster-candidate-5-0-2lchcj\" in resourcegroup=\"rg-candidate-5-0-84dql4hbsnbj\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f71506a-af2f-4b60-985d-00bd566e1074\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f71506a-af2f-4b60-985d-00bd566e1074\",\n      \"name\": \"5f71506a-af2f-4b60-985d-00bd566e1074\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:23:18.647011306Z\",\n      \"endTime\": \"2026-08-25T19:42:23.606424611Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-84dql4hbsnbj/cluster-candidate-5-0-2lchcj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aaf8a0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-2lchcj: failed waiting for cluster=\"cluster-candidate-5-0-2lchcj\" in resourcegroup=\"rg-candidate-5-0-84dql4hbsnbj\" to finish creating: GET https://rp.j6756224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f71506a-af2f-4b60-985d-00bd566e1074\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f71506a-af2f-4b60-985d-00bd566e1074\",\n      \"name\": \"5f71506a-af2f-4b60-985d-00bd566e1074\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:23:18.647011306Z\",\n      \"endTime\": \"2026-08-25T19:42:23.606424611Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-vs25bgt56m2n/cluster-candidate-4-22-mbk6dn should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101e4e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-mbk6dn: failed waiting for cluster=\"cluster-candidate-4-22-mbk6dn\" in resourcegroup=\"rg-candidate-4-22-vs25bgt56m2n\" to finish creating: GET https://rp.j6821248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c94972c0-8210-4d51-960d-183f4fd8e840\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c94972c0-8210-4d51-960d-183f4fd8e840\",\n      \"name\": \"c94972c0-8210-4d51-960d-183f4fd8e840\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:41.789123755Z\",\n      \"endTime\": \"2026-08-25T19:36:42.565693183Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-vs25bgt56m2n/cluster-candidate-4-22-mbk6dn should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101e4e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-mbk6dn: failed waiting for cluster=\"cluster-candidate-4-22-mbk6dn\" in resourcegroup=\"rg-candidate-4-22-vs25bgt56m2n\" to finish creating: GET https://rp.j6821248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c94972c0-8210-4d51-960d-183f4fd8e840\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c94972c0-8210-4d51-960d-183f4fd8e840\",\n      \"name\": \"c94972c0-8210-4d51-960d-183f4fd8e840\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:41.789123755Z\",\n      \"endTime\": \"2026-08-25T19:36:42.565693183Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"91a343c747731a46478c011c994a808882abeec84c6530eb04b1878042920e4e","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316483416756224","occurred_at":"2026-08-25T18:21:37Z","signature_id":"dbe010bd3eef880599620e7f2c75170adabcb8631a1d8b6c5fd42aa908a7aadb","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6682/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314723616821248","occurred_at":"2026-08-25T18:14:37Z","signature_id":"365ceaf0468738d194ab343ea8045834af13aae002f5e53c5e41f490e916f1cc","pr_number":6682,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","signature_id":"25985bba716906e38dc1ab2e46edefe47ee71312f5d30ecca45c85fd1a612dae","pr_number":6630,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.6309148264984227,"anchor_occurrences":4,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[1,1,1,0,1,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":10,"prior_last_seen_at":"2026-08-22T00:26:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010deff0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-4cmmkc\" in resourcegroup=\"admin-credential-lifecycle-test-mk5bt6ghv29k\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\",\n      \"name\": \"0644f278-df05-4603-a10d-3a66a1f851d7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:17:03.672410989Z\",\n      \"endTime\": \"2026-08-24T15:41:10.046088761Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tjfmep0gog9fd4pfkr5bnpc0n642 still exists (deletion dispatched at 2026-08-24T15:17:04Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010deff0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-4cmmkc\" in resourcegroup=\"admin-credential-lifecycle-test-mk5bt6ghv29k\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\",\n      \"name\": \"0644f278-df05-4603-a10d-3a66a1f851d7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:17:03.672410989Z\",\n      \"endTime\": \"2026-08-24T15:41:10.046088761Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tjfmep0gog9fd4pfkr5bnpc0n642 still exists (deletion dispatched at 2026-08-24T15:17:04Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012ab8d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-khp84d8mdsh7\" to finish deleting: GET https://rp.j7976064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b538a56-05be-48db-bb47-4c5a33c4accf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b538a56-05be-48db-bb47-4c5a33c4accf\",\n      \"name\": \"7b538a56-05be-48db-bb47-4c5a33c4accf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:26:12.085661598Z\",\n      \"endTime\": \"2026-08-22T01:50:12.281269064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sbakg7dgt5f0fajefaqb6utb0vjshqh still exists (deletion dispatched at 2026-08-22T01:26:12Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012ab8d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-khp84d8mdsh7\" to finish deleting: GET https://rp.j7976064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b538a56-05be-48db-bb47-4c5a33c4accf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b538a56-05be-48db-bb47-4c5a33c4accf\",\n      \"name\": \"7b538a56-05be-48db-bb47-4c5a33c4accf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:26:12.085661598Z\",\n      \"endTime\": \"2026-08-22T01:50:12.281269064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sbakg7dgt5f0fajefaqb6utb0vjshqh still exists (deletion dispatched at 2026-08-22T01:26:12Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010d41e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-tr4whf2bpfsf\" to finish deleting: GET https://rp.j3372032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e72e998-9d37-429a-9354-c729235c1e6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e72e998-9d37-429a-9354-c729235c1e6f\",\n      \"name\": \"4e72e998-9d37-429a-9354-c729235c1e6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T02:15:23.413243123Z\",\n      \"endTime\": \"2026-08-20T02:39:29.402250383Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa13enfvdgm47tbcg2mg1bln7e4ikq7 still exists (deletion dispatched at 2026-08-20T02:15:25Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010d41e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-tr4whf2bpfsf\" to finish deleting: GET https://rp.j3372032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e72e998-9d37-429a-9354-c729235c1e6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e72e998-9d37-429a-9354-c729235c1e6f\",\n      \"name\": \"4e72e998-9d37-429a-9354-c729235c1e6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T02:15:23.413243123Z\",\n      \"endTime\": \"2026-08-20T02:39:29.402250383Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa13enfvdgm47tbcg2mg1bln7e4ikq7 still exists (deletion dispatched at 2026-08-20T02:15:25Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"7df46629584e4c4253c91624b483f13865f970598c9ed547ff855ac9eea68fe2","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090958673797976064","occurred_at":"2026-08-22T00:26:10Z","signature_id":"b11845b41dfb98943e4686416824e8a527b84d128d07df3276550aa8a60f48ab","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090249792243372032","occurred_at":"2026-08-20T01:29:20Z","signature_id":"413ec903f1148685f58b502ff35e13104028e2eb95ffff3c388e1f10df7b1622","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087196852752384","occurred_at":"2026-08-19T14:43:14Z","signature_id":"2d06bb34613aea8950ff84401b1d039dd84dbc4e5c72eb3633c6ca2db250b20b","pr_number":6608,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.6309148264984227,"anchor_occurrences":4,"after_last_push_count":5,"trend_counts":[0,1,1,1,0,1,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":23,"prior_last_seen_at":"2026-08-23T03:43:19Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","occurrences":4}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-23T06:20:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j7940736-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.111:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7940736/providers/microsoft.monitor/accounts/services-j7940736\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-d5c7cc5b4-wxvwj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j7940736-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-23T06:20:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j7940736-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.111:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7940736/providers/microsoft.monitor/accounts/services-j7940736\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-d5c7cc5b4-wxvwj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j7940736-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T14:05:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j2827648-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.54:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2827648/providers/microsoft.monitor/accounts/services-j2827648\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-67688f45cb-sh5gb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j2827648-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T21:19:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j2508544-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2508544/providers/microsoft.monitor/accounts/services-j2508544\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-68dd4ccf65-vd77g\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j2508544-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T21:19:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j2508544-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2508544/providers/microsoft.monitor/accounts/services-j2508544\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-68dd4ccf65-vd77g\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j2508544-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091370674797940736","occurred_at":"2026-08-23T03:43:19Z","signature_id":"f1da0e02131d79f9d743173dac73f72f60ddab0fc62aa516115c813c4011a54f","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"646de8fc5dad2d940ca1d6f85fbd453d41a6658308be00df36dea55382f0f44c","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"58c618af7602c927c3c11c14a7755006e83ca1016ea028f137f644807b446b83","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"1566a875249dafa2e6a8224e300d970974f75796f4a571cd360e9836b7e23199","pr_number":6601,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"34e193d1e971458756c5fedbfac4e10a25751030aa42d3492b58c65f0ceda0fd","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.47318611987381703,"anchor_occurrences":4,"after_last_push_count":1,"also_in":["PROD","STG"],"trend_counts":[0,0,0,0,0,0,2,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-10"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-14T20:39:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-pg2742k6drr6/cluster-candidate-4-20-s9697v should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00121c780\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-s9697v: failed waiting for cluster=\"cluster-candidate-4-20-s9697v\" in resourcegroup=\"rg-candidate-4-20-pg2742k6drr6\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18bd3e8a-96dd-4c6e-9e86-32f76567eee8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18bd3e8a-96dd-4c6e-9e86-32f76567eee8\",\n      \"name\": \"18bd3e8a-96dd-4c6e-9e86-32f76567eee8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T15:26:38.293191515Z\",\n      \"endTime\": \"2026-08-25T15:45:40.878934392Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-pg2742k6drr6/cluster-candidate-4-20-s9697v should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00121c780\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-s9697v: failed waiting for cluster=\"cluster-candidate-4-20-s9697v\" in resourcegroup=\"rg-candidate-4-20-pg2742k6drr6\" to finish creating: GET https://rp.j8947456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18bd3e8a-96dd-4c6e-9e86-32f76567eee8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18bd3e8a-96dd-4c6e-9e86-32f76567eee8\",\n      \"name\": \"18bd3e8a-96dd-4c6e-9e86-32f76567eee8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T15:26:38.293191515Z\",\n      \"endTime\": \"2026-08-25T15:45:40.878934392Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000706780\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-jnnzd8dv8sc2\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4c5630e-4920-46ba-ba8e-4a4b51caf88c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4c5630e-4920-46ba-ba8e-4a4b51caf88c\",\n      \"name\": \"d4c5630e-4920-46ba-ba8e-4a4b51caf88c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:53.022759327Z\",\n      \"endTime\": \"2026-08-25T11:22:55.91053294Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000706780\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-jnnzd8dv8sc2\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4c5630e-4920-46ba-ba8e-4a4b51caf88c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4c5630e-4920-46ba-ba8e-4a4b51caf88c\",\n      \"name\": \"d4c5630e-4920-46ba-ba8e-4a4b51caf88c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:53.022759327Z\",\n      \"endTime\": \"2026-08-25T11:22:55.91053294Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:304]: failed to create HCP cluster np-noedge-7pzvlw\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e28760\u003e: \n    failed to create HCP cluster np-noedge-7pzvlw: failed waiting for cluster=\"np-noedge-7pzvlw\" in resourcegroup=\"rg-np-noedge-7pzvlw-hmhh49x6v5k9\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n      \"name\": \"d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:29:01.361816726Z\",\n      \"endTime\": \"2026-08-24T21:48:06.132587418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:304]: failed to create HCP cluster np-noedge-7pzvlw\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e28760\u003e: \n    failed to create HCP cluster np-noedge-7pzvlw: failed waiting for cluster=\"np-noedge-7pzvlw\" in resourcegroup=\"rg-np-noedge-7pzvlw-hmhh49x6v5k9\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n      \"name\": \"d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:29:01.361816726Z\",\n      \"endTime\": \"2026-08-24T21:48:06.132587418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"fe4f032da53c9249c0554d3122ef5a19cf72dc9f02c7bcd2d72076136ebe7044","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"9021161863b438a4babb0dca655669309db8e7ffa0c5f505c949b37f07510077","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"7467fcd7d827463de8cbe0d7b5f551f437c25ba6ae7fc7de2730705826346049","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"ca98190d0c65ef7e152d8e004d3f56a8bcff3a1f40939b19e5d08cfc9a992c4b","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.47318611987381703,"anchor_occurrences":4,"after_last_push_count":1,"trend_counts":[0,2,2,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-20T19:00:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0017e29c0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.33) failed: no version in clusterversion/version status.history is greater than initial \"4.20.33\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0017e29c0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.33) failed: no version in clusterversion/version status.history is greater than initial \"4.20.33\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00120a5b8\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.28) failed: no version in clusterversion/version status.history is greater than initial \"4.21.28\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00120a5b8\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.28) failed: no version in clusterversion/version status.history is greater than initial \"4.21.28\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000883788\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.33) failed: no version in clusterversion/version status.history is greater than initial \"4.20.33\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000883788\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.33) failed: no version in clusterversion/version status.history is greater than initial \"4.20.33\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"1e125fa309bbfccb11249a0675fd870ebfb9e84ba21b179b4396c1b5598d7407","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"278e50384d73aa823a12869e8d59b4d21af4c3d554b9dfe27062a944fd0b322d","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"9cc8145e1eea121a8e0d1d77fe937aa6850e73bb34ffac96adf809bdcb76deb6","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"3ce6644c62e65c2c36bdbcc4461f08f990d2eba9f705a686f886c81cf4554eb5","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b889d809a654deaf1fcd0d6bef27feaf62c1f89d458297965bd85e6944d16a40","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,4],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-17T17:42:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cd4b20\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7nfq89d68b2j\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b32b4b1-4670-40ea-8086-f9466f12ce3e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b32b4b1-4670-40ea-8086-f9466f12ce3e\",\n      \"name\": \"6b32b4b1-4670-40ea-8086-f9466f12ce3e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:06.362546319Z\",\n      \"endTime\": \"2026-08-25T11:24:07.656848362Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, openshift-controller-manager, cluster-node-tuning-operator, community-operators-catalog, olm-operator, cluster-storage-operator, cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, catalog-operator, redhat-marketplace-catalog, cluster-policy-controller, packageserver, dns-operator, ingress-operator, cluster-image-registry-operator, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cd4b20\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7nfq89d68b2j\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b32b4b1-4670-40ea-8086-f9466f12ce3e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b32b4b1-4670-40ea-8086-f9466f12ce3e\",\n      \"name\": \"6b32b4b1-4670-40ea-8086-f9466f12ce3e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:06.362546319Z\",\n      \"endTime\": \"2026-08-25T11:24:07.656848362Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, openshift-controller-manager, cluster-node-tuning-operator, community-operators-catalog, olm-operator, cluster-storage-operator, cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, catalog-operator, redhat-marketplace-catalog, cluster-policy-controller, packageserver, dns-operator, ingress-operator, cluster-image-registry-operator, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001116340\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-bjqmncjbznvt\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64e0d347-b695-455c-b828-66deb96d7452\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64e0d347-b695-455c-b828-66deb96d7452\",\n      \"name\": \"64e0d347-b695-455c-b828-66deb96d7452\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:05.579389204Z\",\n      \"endTime\": \"2026-08-25T11:24:07.68094221Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, olm-operator, openshift-route-controller-manager, openshift-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, catalog-operator, certified-operators-catalog, packageserver, cluster-network-operator, ingress-operator, cluster-version-operator, openshift-apiserver, dns-operator, cluster-policy-controller, redhat-marketplace-catalog, community-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001116340\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-bjqmncjbznvt\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64e0d347-b695-455c-b828-66deb96d7452\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/64e0d347-b695-455c-b828-66deb96d7452\",\n      \"name\": \"64e0d347-b695-455c-b828-66deb96d7452\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:05.579389204Z\",\n      \"endTime\": \"2026-08-25T11:24:07.68094221Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, olm-operator, openshift-route-controller-manager, openshift-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, catalog-operator, certified-operators-catalog, packageserver, cluster-network-operator, ingress-operator, cluster-version-operator, openshift-apiserver, dns-operator, cluster-policy-controller, redhat-marketplace-catalog, community-operators-catalog, hosted-cluster-config-operator, cluster-storage-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f38340\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-4zrvgxxs9sbf\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\",\n      \"name\": \"b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:16.036728583Z\",\n      \"endTime\": \"2026-08-25T11:24:17.88745643Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, openshift-route-controller-manager, redhat-operators-catalog, olm-operator, community-operators-catalog, cluster-policy-controller, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-storage-operator, packageserver, hosted-cluster-config-operator, ingress-operator, csi-snapshot-controller-operator, openshift-controller-manager, certified-operators-catalog, cluster-version-operator, cluster-network-operator, dns-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f38340\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-4zrvgxxs9sbf\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\",\n      \"name\": \"b9cf654f-aed9-4e51-9d7f-39b0aaef1fa4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:16.036728583Z\",\n      \"endTime\": \"2026-08-25T11:24:17.88745643Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, openshift-route-controller-manager, redhat-operators-catalog, olm-operator, community-operators-catalog, cluster-policy-controller, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-storage-operator, packageserver, hosted-cluster-config-operator, ingress-operator, csi-snapshot-controller-operator, openshift-controller-manager, certified-operators-catalog, cluster-version-operator, cluster-network-operator, dns-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"15cec8044c988cc97a4d3ec832848e5522b08071d86565ff2ec35915e8e8e5ea","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"8e4e26a914a87830fbedc93c1a377a82ad141c9089c89b9edbacc2b4d4d5a21f","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"a6d52fac7766cec3d727e2f0d94855f4d0b82cda86272b0009927fee77388e73","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"0502e7bea814f2dea497a2fefaef20aef7c9a61fc571ac4894bd795f56d24e38","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4a2f8bcf8d29c66434783789d75f6d18e6e94158f0e0f35e53c0353bce508345","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":4,"after_last_push_count":3,"also_in":["STG"],"trend_counts":[0,2,0,0,0,0,2,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-19T18:19:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e70e20\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wmkndtkx988t\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\",\n      \"name\": \"a9d4baef-549e-4561-b748-439453d113af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:49:08.39871397Z\",\n      \"endTime\": \"2026-08-24T16:08:11.236941815Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e70e20\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wmkndtkx988t\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\",\n      \"name\": \"a9d4baef-549e-4561-b748-439453d113af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:49:08.39871397Z\",\n      \"endTime\": \"2026-08-24T16:08:11.236941815Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000408ec0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bn9cdfvxcqll\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\",\n      \"name\": \"75726762-be88-402f-a564-60c91125dc8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:47:28.10624954Z\",\n      \"endTime\": \"2026-08-24T16:06:31.149377863Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000408ec0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bn9cdfvxcqll\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\",\n      \"name\": \"75726762-be88-402f-a564-60c91125dc8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:47:28.10624954Z\",\n      \"endTime\": \"2026-08-24T16:06:31.149377863Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e70880\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-x6prxbl8v6bk\" to finish creating: GET https://rp.j1797504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2382ad9-6776-48ee-9bf9-e8c510952cb6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2382ad9-6776-48ee-9bf9-e8c510952cb6\",\n      \"name\": \"e2382ad9-6776-48ee-9bf9-e8c510952cb6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:08:40.647335859Z\",\n      \"endTime\": \"2026-08-19T19:27:47.028573359Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e70880\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-x6prxbl8v6bk\" to finish creating: GET https://rp.j1797504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2382ad9-6776-48ee-9bf9-e8c510952cb6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2382ad9-6776-48ee-9bf9-e8c510952cb6\",\n      \"name\": \"e2382ad9-6776-48ee-9bf9-e8c510952cb6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:08:40.647335859Z\",\n      \"endTime\": \"2026-08-19T19:27:47.028573359Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"7dd423d6ebe7dc246942b7ca078647aad56aaec8baea7f2d4aa8292a909f5f19","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"d07feba14b1358e1ba91e00ab3fcc20b89813cd03b2520364e81ecc050b70a83","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"5249410476dac4a69075b89f7eafcada175646a0b2ec9eb6125986a39843aef1","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"6cc6d9075dc9c54a3a94cfefb4e9f32ae722fa4876cd14876ed318bd9d803a1e","pr_number":6611,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.47318611987381703,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,2,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-19T09:11:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-fn4s2m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dea2e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-fn4s2m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dea2e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:131]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005da4a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:131]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005da4a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-rp2lrf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b70e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-r92l5.apps.aro.h0k3z1f6y5f5g2r.rlny.j0859904.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-r92l5.apps.aro.h0k3z1f6y5f5g2r.rlny.j0859904.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-r92l5.apps.aro.h0k3z1f6y5f5g2r.rlny.j0859904.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-rp2lrf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b70e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-r92l5.apps.aro.h0k3z1f6y5f5g2r.rlny.j0859904.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-r92l5.apps.aro.h0k3z1f6y5f5g2r.rlny.j0859904.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-r92l5.apps.aro.h0k3z1f6y5f5g2r.rlny.j0859904.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977495384428544","occurred_at":"2026-08-24T19:54:36Z","signature_id":"4a86fd68539d0c65cc2c794f0a3b117d62969a34ae1b535d87cc0cce6d88b528","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091811921836642304","occurred_at":"2026-08-24T08:56:40Z","signature_id":"67e0d9c3e541426a700c2b1097cab5407e75837246f0b2492c0fa4c512883fb9","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090003636900859904","occurred_at":"2026-08-19T09:11:12Z","signature_id":"9094f8b2d47d1b3fb610e6e3fed35fc3383395672932f3de5845d46d2dbfa706","pr_number":6601,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6fc3be35c0207564b1dc5633bee2c60ed2e3e90031e5c78b08b3a7bcef7800ec","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.47318611987381703,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ddcf40\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-q2q7d86xnzgs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\",\n      \"name\": \"97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:18:39.021640766Z\",\n      \"endTime\": \"2026-08-25T19:37:42.379207389Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ddcf40\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-q2q7d86xnzgs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\",\n      \"name\": \"97e1f38f-9a8a-49c5-a5bb-7aeee1ce49a6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:18:39.021640766Z\",\n      \"endTime\": \"2026-08-25T19:37:42.379207389Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-c4b48j\" with version 4.21.29 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001892580\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-c4b48j: failed waiting for cluster=\"cluster-zstream-4-21-c4b48j\" in resourcegroup=\"rg-zstream-upgrade-4-21-tjwbnczmk66w\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dd2277c-c818-44a4-91cb-db69efb21df4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dd2277c-c818-44a4-91cb-db69efb21df4\",\n      \"name\": \"5dd2277c-c818-44a4-91cb-db69efb21df4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:24:22.258032156Z\",\n      \"endTime\": \"2026-08-25T16:43:28.791772827Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-c4b48j\" with version 4.21.29 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001892580\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-c4b48j: failed waiting for cluster=\"cluster-zstream-4-21-c4b48j\" in resourcegroup=\"rg-zstream-upgrade-4-21-tjwbnczmk66w\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dd2277c-c818-44a4-91cb-db69efb21df4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dd2277c-c818-44a4-91cb-db69efb21df4\",\n      \"name\": \"5dd2277c-c818-44a4-91cb-db69efb21df4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:24:22.258032156Z\",\n      \"endTime\": \"2026-08-25T16:43:28.791772827Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-zgp8947jk9nr/cluster-candidate-4-22-8d2tgf should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e64200\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-8d2tgf: failed waiting for cluster=\"cluster-candidate-4-22-8d2tgf\" in resourcegroup=\"rg-candidate-4-22-zgp8947jk9nr\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8fe159a5-0698-4edb-aeeb-bac8baa0530e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8fe159a5-0698-4edb-aeeb-bac8baa0530e\",\n      \"name\": \"8fe159a5-0698-4edb-aeeb-bac8baa0530e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:29.272254875Z\",\n      \"endTime\": \"2026-08-25T11:21:35.930400552Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-zgp8947jk9nr/cluster-candidate-4-22-8d2tgf should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e64200\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-8d2tgf: failed waiting for cluster=\"cluster-candidate-4-22-8d2tgf\" in resourcegroup=\"rg-candidate-4-22-zgp8947jk9nr\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8fe159a5-0698-4edb-aeeb-bac8baa0530e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8fe159a5-0698-4edb-aeeb-bac8baa0530e\",\n      \"name\": \"8fe159a5-0698-4edb-aeeb-bac8baa0530e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:29.272254875Z\",\n      \"endTime\": \"2026-08-25T11:21:35.930400552Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"84131327fae01288ea8861e4a967044b615e020a15959c0d9a099127d951ae13","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269882870599680","occurred_at":"2026-08-25T15:16:27Z","signature_id":"4cde9318d023ca11a0bb6154c219327b20248fc8365ff58ea50734ae7b25de59","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"541a9bf4087000257fc777c8bd4137d002f9e23cd4994e52825872d346f25bdb","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"30d5e903df58307358d5dbc7b4047e4f537ca0b7b5e66487f298253e6e91fb1f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.47318611987381703,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-17T22:25:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001c0e640\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-58h6tmctnmgt\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0aad29d-cf26-4c83-b0f3-4c189b3e6419\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0aad29d-cf26-4c83-b0f3-4c189b3e6419\",\n      \"name\": \"d0aad29d-cf26-4c83-b0f3-4c189b3e6419\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:19:02.364791294Z\",\n      \"endTime\": \"2026-08-25T19:38:02.432181787Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001c0e640\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-58h6tmctnmgt\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0aad29d-cf26-4c83-b0f3-4c189b3e6419\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0aad29d-cf26-4c83-b0f3-4c189b3e6419\",\n      \"name\": \"d0aad29d-cf26-4c83-b0f3-4c189b3e6419\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:19:02.364791294Z\",\n      \"endTime\": \"2026-08-25T19:38:02.432181787Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b67620\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-wgbj289pcnlt\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37f858c7-a07c-43e4-a020-589372bc5756\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37f858c7-a07c-43e4-a020-589372bc5756\",\n      \"name\": \"37f858c7-a07c-43e4-a020-589372bc5756\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:54:06.405705309Z\",\n      \"endTime\": \"2026-08-25T15:13:09.352514019Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b67620\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-wgbj289pcnlt\" to finish creating: GET https://rp.j4024704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37f858c7-a07c-43e4-a020-589372bc5756\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37f858c7-a07c-43e4-a020-589372bc5756\",\n      \"name\": \"37f858c7-a07c-43e4-a020-589372bc5756\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:54:06.405705309Z\",\n      \"endTime\": \"2026-08-25T15:13:09.352514019Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-6nw9hm with version 4.20.35\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000892700\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-6nw9hm: failed waiting for cluster=\"np-version-upgrade-cluster-6nw9hm\" in resourcegroup=\"rg-np-version-upgrade-6nw9hm-sbc4vzg69snt\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd8368f0-8487-43cb-9ffb-d72a8e31a349\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd8368f0-8487-43cb-9ffb-d72a8e31a349\",\n      \"name\": \"dd8368f0-8487-43cb-9ffb-d72a8e31a349\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:13:18.056611915Z\",\n      \"endTime\": \"2026-08-25T11:32:25.82846965Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-6nw9hm with version 4.20.35\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000892700\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-6nw9hm: failed waiting for cluster=\"np-version-upgrade-cluster-6nw9hm\" in resourcegroup=\"rg-np-version-upgrade-6nw9hm-sbc4vzg69snt\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd8368f0-8487-43cb-9ffb-d72a8e31a349\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd8368f0-8487-43cb-9ffb-d72a8e31a349\",\n      \"name\": \"dd8368f0-8487-43cb-9ffb-d72a8e31a349\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:13:18.056611915Z\",\n      \"endTime\": \"2026-08-25T11:32:25.82846965Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"6873e149feb92978c0e6c51d500f8132748dcf6414954a9cbceb9d163428d9a0","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092253165134024704","occurred_at":"2026-08-25T14:10:01Z","signature_id":"9f0220de1cfd60808a1dbdb143dba6ec11917303d1f81188a2549135945ac704","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"d833ff037f5488f1fab9b945d4f55d14924d6973d61766c5187da39c788703a1","pr_number":6618,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"281d6f2e10296c7900f78767c52e72d480329fd4f2572568eaae28e31a3fe696","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:48700-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.47318611987381703,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,1,2,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-10","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-20T17:02:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:137]: failed to verify basic cluster health for idms-e2e-hcp-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc0010a5830\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:48700-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:48700-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:137]: failed to verify basic cluster health for idms-e2e-hcp-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc0010a5830\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:48700-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:48700-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:241]: failed to verify HCP cluster viability\nUnexpected error:\n    \u003c*errors.joinError | 0xc00113a678\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.f8q0u2e6o9u3o4v.60na.j3674112.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.f8q0u2e6o9u3o4v.60na.j3674112.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.130.38.9:35166-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.f8q0u2e6o9u3o4v.60na.j3674112.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.f8q0u2e6o9u3o4v.60na.j3674112.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.130.38.9:35166-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:241]: failed to verify HCP cluster viability\nUnexpected error:\n    \u003c*errors.joinError | 0xc00113a678\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.f8q0u2e6o9u3o4v.60na.j3674112.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.f8q0u2e6o9u3o4v.60na.j3674112.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.130.38.9:35166-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.f8q0u2e6o9u3o4v.60na.j3674112.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.f8q0u2e6o9u3o4v.60na.j3674112.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.130.38.9:35166-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:135]: failed to verify HCP cluster \"cluster-candidate-5-0-6b6mzt\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc000f25020\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.v7y4d9w4j7z1d4r.ptmj.j2752384.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.v7y4d9w4j7z1d4r.ptmj.j2752384.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.128.36.17:43656-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.v7y4d9w4j7z1d4r.ptmj.j2752384.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.v7y4d9w4j7z1d4r.ptmj.j2752384.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.128.36.17:43656-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:135]: failed to verify HCP cluster \"cluster-candidate-5-0-6b6mzt\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc000f25020\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.v7y4d9w4j7z1d4r.ptmj.j2752384.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.v7y4d9w4j7z1d4r.ptmj.j2752384.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.128.36.17:43656-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.v7y4d9w4j7z1d4r.ptmj.j2752384.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.v7y4d9w4j7z1d4r.ptmj.j2752384.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.128.36.17:43656-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090484744067223552","occurred_at":"2026-08-20T17:02:48Z","signature_id":"7648b78127bfdef7b2e7f4fb0449c332b1c8467f57775d688c9bd8ee54b5f2b4","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248642563674112","occurred_at":"2026-08-20T01:24:45Z","signature_id":"e9186488e62d93403aa4b53aad2c98b53df0c580027391c25bdf01e79761323c","pr_number":6615,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087196852752384","occurred_at":"2026-08-19T14:43:14Z","signature_id":"8449f561e1d8c267e5688dd44ff0e20f4db56086e1dd6074eee4fe043bf86004","pr_number":6608,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b952bd87cc064f775e96c64b34d1d595213c1b657abf569b98d2c49605644533","failure_pattern":"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4","search_query":"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.47318611987381703,"anchor_occurrences":3,"after_last_push_count":6,"trend_counts":[4,1,1,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":19,"prior_last_seen_at":"2026-08-21T21:20:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:194]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0011f9ad0\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:194]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0011f9ad0\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f8a348\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f8a348\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001a2ab10\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001a2ab10\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912019128520704","occurred_at":"2026-08-21T21:20:47Z","signature_id":"105cc56e52a2d552e4ddbe874bd058b495386714220d4a05879a34d8129b2709","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"7b88e73ef704bd58321b40735ba98a518a526cd77c56eddfbc7f78f59d7dfb88","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"e4ce4e490406651d21d2d7a2c92ef7cf75c01173be981a5167bf0f960c11f06a","pr_number":6601,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"95c96916264bf9de45b37005d736b6d9552404339f828cf7083033bb2290c13f","failure_pattern":"alert [svc] FleetControllerRetryHotLoop fired","search_query":"alert [svc] FleetControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.47318611987381703,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[3,1,0,0,0,0,1,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-19T22:05:11Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T15:28:49Z\n  Ended:       2026-08-25T15:35:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"FleetControllerRetryHotLoop\", cluster=\"ci01-j0144640-svc\", component=\"fleet\", name=\"clustersserviceregistrationcontroller\", severity=\"warning\"\n  Description: Fleet controller workqueue clustersserviceregistrationcontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T17:38:07Z\n  Ended:       2026-08-24T17:45:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"FleetControllerRetryHotLoop\", cluster=\"ci01-j0629888-svc\", component=\"fleet\", name=\"clustersserviceregistrationcontroller\", severity=\"warning\"\n  Description: Fleet controller workqueue clustersserviceregistrationcontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-19T22:56:27Z\n  Ended:       2026-08-19T23:03:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"FleetControllerRetryHotLoop\", cluster=\"ci01-j2195200-svc\", component=\"fleet\", name=\"clustersserviceregistrationcontroller\", severity=\"warning\"\n  Description: Fleet controller workqueue clustersserviceregistrationcontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092258099980144640","occurred_at":"2026-08-25T14:29:37Z","signature_id":"3eebb3220fdafc37e59ae5d05cda26d2e052d0d1681fedb42f8fa69446913217","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"3eebb3220fdafc37e59ae5d05cda26d2e052d0d1681fedb42f8fa69446913217","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090198433062195200","occurred_at":"2026-08-19T22:05:11Z","signature_id":"3eebb3220fdafc37e59ae5d05cda26d2e052d0d1681fedb42f8fa69446913217","pr_number":6492,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.47318611987381703,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[0,2,0,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":15,"prior_last_seen_at":"2026-08-19T16:39:13Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1}],"full_error_samples":["time=2026-08-24T09:44:43.919Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T09:44:44.928Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-08-24T09:44:46.104Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=f40d7b7ab0b2d9646dc8b78fbaa252859442c0d4d4549c8765d66346f06ac123 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4585344-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff40d7b7ab0b2d9646dc8b78fbaa252859442c0d4d4549c8765d66346f06ac123\ntime=2026-08-24T10:14:43.424Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-08-19T16:52:19.566Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-19T16:52:21.760Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-19T16:52:23.532Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=b29ce57d48a551877ae83bcdc41f601e4ab16f4893b46fbfe80ac0981dd4dfea portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9953536-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fb29ce57d48a551877ae83bcdc41f601e4ab16f4893b46fbfe80ac0981dd4dfea\ntime=2026-08-19T17:22:19.278Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-08-19T12:23:07.894Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-19T12:23:08.895Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-08-19T12:23:10.111Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=4ab82aee364a2c4f78d1f63c483aaade6c8eb3341b871d39ccb8291832806f35 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4234112-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F4ab82aee364a2c4f78d1f63c483aaade6c8eb3341b871d39ccb8291832806f35\ntime=2026-08-19T12:53:07.512Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6655/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091810349014585344","occurred_at":"2026-08-24T08:50:25Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6655,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090116384229953536","occurred_at":"2026-08-19T16:39:13Z","signature_id":"546713f253dbb07bdf6d6b240d468c73b1927023c4bbf21dee04e87aaa304e9a","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090044954054234112","occurred_at":"2026-08-19T11:55:22Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6547,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1991ea49f3ca2d6d94a2581eca784f9c57feb75f5c693bb159399a0906185730","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cfa540\u003e: \n    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-6b79jbxxk2jv\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51cea064-8c25-47d5-9e4c-98159ead62b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51cea064-8c25-47d5-9e4c-98159ead62b6\",\n      \"name\": \"51cea064-8c25-47d5-9e4c-98159ead62b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:45.544982989Z\",\n      \"endTime\": \"2026-08-25T11:23:47.560554329Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, cluster-version-operator, cluster-network-operator, olm-operator, cluster-storage-operator, redhat-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, ingress-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, catalog-operator, cluster-policy-controller, cluster-node-tuning-operator, dns-operator, certified-operators-catalog, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-image-registry-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cfa540\u003e: \n    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-6b79jbxxk2jv\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51cea064-8c25-47d5-9e4c-98159ead62b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51cea064-8c25-47d5-9e4c-98159ead62b6\",\n      \"name\": \"51cea064-8c25-47d5-9e4c-98159ead62b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:45.544982989Z\",\n      \"endTime\": \"2026-08-25T11:23:47.560554329Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, cluster-version-operator, cluster-network-operator, olm-operator, cluster-storage-operator, redhat-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, ingress-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, catalog-operator, cluster-policy-controller, cluster-node-tuning-operator, dns-operator, certified-operators-catalog, hosted-cluster-config-operator, azure-cloud-controller-manager, cluster-image-registry-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-wmc2th\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c80de0\u003e: \n    failed to create HCP cluster np-noedge-wmc2th: failed waiting for cluster=\"np-noedge-wmc2th\" in resourcegroup=\"rg-np-noedge-wmc2th-55l76kt7fdfw\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/599e72ae-426e-413c-a454-0c456b2c46e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/599e72ae-426e-413c-a454-0c456b2c46e6\",\n      \"name\": \"599e72ae-426e-413c-a454-0c456b2c46e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:46.955311835Z\",\n      \"endTime\": \"2026-08-25T11:23:47.543149183Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-apiserver, openshift-controller-manager, hosted-cluster-config-operator, ingress-operator, packageserver, cluster-version-operator, cluster-policy-controller, cluster-storage-operator, dns-operator, olm-operator, cluster-image-registry-operator, redhat-operators-catalog, community-operators-catalog, azure-cloud-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, certified-operators-catalog, cluster-network-operator, catalog-operator, redhat-marketplace-catalog, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-wmc2th\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c80de0\u003e: \n    failed to create HCP cluster np-noedge-wmc2th: failed waiting for cluster=\"np-noedge-wmc2th\" in resourcegroup=\"rg-np-noedge-wmc2th-55l76kt7fdfw\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/599e72ae-426e-413c-a454-0c456b2c46e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/599e72ae-426e-413c-a454-0c456b2c46e6\",\n      \"name\": \"599e72ae-426e-413c-a454-0c456b2c46e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:46.955311835Z\",\n      \"endTime\": \"2026-08-25T11:23:47.543149183Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-apiserver, openshift-controller-manager, hosted-cluster-config-operator, ingress-operator, packageserver, cluster-version-operator, cluster-policy-controller, cluster-storage-operator, dns-operator, olm-operator, cluster-image-registry-operator, redhat-operators-catalog, community-operators-catalog, azure-cloud-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, certified-operators-catalog, cluster-network-operator, catalog-operator, redhat-marketplace-catalog, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00134ebc0\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-7p6xhpjkj2d7\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9b87905-c782-40c9-95b4-fab4bb1cb211\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9b87905-c782-40c9-95b4-fab4bb1cb211\",\n      \"name\": \"e9b87905-c782-40c9-95b4-fab4bb1cb211\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:49.498742711Z\",\n      \"endTime\": \"2026-08-25T11:23:52.96330385Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, ingress-operator, cluster-policy-controller, dns-operator, catalog-operator, openshift-route-controller-manager, cluster-node-tuning-operator, redhat-operators-catalog, cluster-storage-operator, cluster-network-operator, cluster-image-registry-operator, packageserver, azure-cloud-controller-manager, openshift-controller-manager, certified-operators-catalog, olm-operator, redhat-marketplace-catalog, hosted-cluster-config-operator, csi-snapshot-controller-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00134ebc0\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-7p6xhpjkj2d7\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9b87905-c782-40c9-95b4-fab4bb1cb211\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9b87905-c782-40c9-95b4-fab4bb1cb211\",\n      \"name\": \"e9b87905-c782-40c9-95b4-fab4bb1cb211\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:49.498742711Z\",\n      \"endTime\": \"2026-08-25T11:23:52.96330385Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, ingress-operator, cluster-policy-controller, dns-operator, catalog-operator, openshift-route-controller-manager, cluster-node-tuning-operator, redhat-operators-catalog, cluster-storage-operator, cluster-network-operator, cluster-image-registry-operator, packageserver, azure-cloud-controller-manager, openshift-controller-manager, certified-operators-catalog, olm-operator, redhat-marketplace-catalog, hosted-cluster-config-operator, csi-snapshot-controller-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"7962adc60e87afd58df615180673875e6038c2e6c21e761c06a8e2250c897c04","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"baf1c4495aac4315f07ab15d045d6bf7c36c718b7b6f2252cb6b1dd46b38088c","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"4ca6751eeab4072d0b147aaed4435bad054118427342ce0f828034310536b0c2","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"13af4eebb33ef90402931e7ab13148cdff6daac3fb3b395a1841590fdaff6d91","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e94fa0\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-w7gfkxf8nw4v\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8aa0882c-9021-414b-b83f-d1f980679679\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8aa0882c-9021-414b-b83f-d1f980679679\",\n      \"name\": \"8aa0882c-9021-414b-b83f-d1f980679679\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:49.25563444Z\",\n      \"endTime\": \"2026-08-25T19:36:50.662196769Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-network-operator, ingress-operator, packageserver, dns-operator, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e94fa0\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-w7gfkxf8nw4v\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8aa0882c-9021-414b-b83f-d1f980679679\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8aa0882c-9021-414b-b83f-d1f980679679\",\n      \"name\": \"8aa0882c-9021-414b-b83f-d1f980679679\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:49.25563444Z\",\n      \"endTime\": \"2026-08-25T19:36:50.662196769Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-network-operator, ingress-operator, packageserver, dns-operator, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001d5620\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-nt76rfflxn6j\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01fbd967-26b2-47f3-ac40-3c302a3e7965\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01fbd967-26b2-47f3-ac40-3c302a3e7965\",\n      \"name\": \"01fbd967-26b2-47f3-ac40-3c302a3e7965\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:49.227475903Z\",\n      \"endTime\": \"2026-08-25T19:36:50.710544904Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, cluster-network-operator, dns-operator, ingress-operator, packageserver, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001d5620\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-nt76rfflxn6j\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01fbd967-26b2-47f3-ac40-3c302a3e7965\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01fbd967-26b2-47f3-ac40-3c302a3e7965\",\n      \"name\": \"01fbd967-26b2-47f3-ac40-3c302a3e7965\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:49.227475903Z\",\n      \"endTime\": \"2026-08-25T19:36:50.710544904Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, cluster-network-operator, dns-operator, ingress-operator, packageserver, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-qclt2j\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012c2120\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-qclt2j: failed waiting for cluster=\"cp-ystream-upgrade-4-21-qclt2j\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-2mbkqtdbwbbj\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c79cbefa-a0f4-4710-a616-f42a2f2efab3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c79cbefa-a0f4-4710-a616-f42a2f2efab3\",\n      \"name\": \"c79cbefa-a0f4-4710-a616-f42a2f2efab3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:09.836508695Z\",\n      \"endTime\": \"2026-08-25T14:42:13.731155696Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, dns-operator, csi-snapshot-controller-operator, cluster-storage-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:147]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-qclt2j\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012c2120\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-qclt2j: failed waiting for cluster=\"cp-ystream-upgrade-4-21-qclt2j\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-2mbkqtdbwbbj\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c79cbefa-a0f4-4710-a616-f42a2f2efab3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c79cbefa-a0f4-4710-a616-f42a2f2efab3\",\n      \"name\": \"c79cbefa-a0f4-4710-a616-f42a2f2efab3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:09.836508695Z\",\n      \"endTime\": \"2026-08-25T14:42:13.731155696Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, dns-operator, csi-snapshot-controller-operator, cluster-storage-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"4e3c6fa2936ba28e83ab9d88f6267d56844fa246b84839315cd5a1f94e2ab484","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"73aa3f2260ddb71b0e4372ee5ba731cfabeb239d1e551a06c5cc7e6c31570148","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"26cd31b13da1cd4b784cc2fc7816f909f73ac2fec28631501c0c3e5e8645ce51","pr_number":6641,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b25a155e232dfc779d7fd5ed5fec6315418c0f2c2aacc2afd101b975cf930003","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, dns-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--w5dw7vtzvxkx/cluster-hs-pre-dlbp6v should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1e760\u003e: \n    failed to create HCP cluster cluster-hs-pre-dlbp6v: failed waiting for cluster=\"cluster-hs-pre-dlbp6v\" in resourcegroup=\"rg-hs-pre--w5dw7vtzvxkx\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7246b37-b59e-4bc0-9719-b3713bc167b2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7246b37-b59e-4bc0-9719-b3713bc167b2\",\n      \"name\": \"f7246b37-b59e-4bc0-9719-b3713bc167b2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:47.341416632Z\",\n      \"endTime\": \"2026-08-25T19:36:50.700032947Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--w5dw7vtzvxkx/cluster-hs-pre-dlbp6v should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1e760\u003e: \n    failed to create HCP cluster cluster-hs-pre-dlbp6v: failed waiting for cluster=\"cluster-hs-pre-dlbp6v\" in resourcegroup=\"rg-hs-pre--w5dw7vtzvxkx\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7246b37-b59e-4bc0-9719-b3713bc167b2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7246b37-b59e-4bc0-9719-b3713bc167b2\",\n      \"name\": \"f7246b37-b59e-4bc0-9719-b3713bc167b2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:17:47.341416632Z\",\n      \"endTime\": \"2026-08-25T19:36:50.700032947Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-c2f59xczgxh8/cluster-candidate-4-21-57drd2 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001bba960\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-57drd2: failed waiting for cluster=\"cluster-candidate-4-21-57drd2\" in resourcegroup=\"rg-candidate-4-21-c2f59xczgxh8\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8baa3d8-e441-4aa4-ad34-789f804408c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8baa3d8-e441-4aa4-ad34-789f804408c7\",\n      \"name\": \"d8baa3d8-e441-4aa4-ad34-789f804408c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:18.423236357Z\",\n      \"endTime\": \"2026-08-25T11:21:25.878162329Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-c2f59xczgxh8/cluster-candidate-4-21-57drd2 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001bba960\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-57drd2: failed waiting for cluster=\"cluster-candidate-4-21-57drd2\" in resourcegroup=\"rg-candidate-4-21-c2f59xczgxh8\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8baa3d8-e441-4aa4-ad34-789f804408c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8baa3d8-e441-4aa4-ad34-789f804408c7\",\n      \"name\": \"d8baa3d8-e441-4aa4-ad34-789f804408c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:18.423236357Z\",\n      \"endTime\": \"2026-08-25T11:21:25.878162329Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-k9tccs with version 4.21.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005c61e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-k9tccs: failed waiting for cluster=\"np-version-upgrade-cluster-k9tccs\" in resourcegroup=\"rg-np-version-upgrade-k9tccs-dxkthfx2xcmq\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06b6d562-cc76-49e7-90e9-e0c893ebdaca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06b6d562-cc76-49e7-90e9-e0c893ebdaca\",\n      \"name\": \"06b6d562-cc76-49e7-90e9-e0c893ebdaca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:30.260205158Z\",\n      \"endTime\": \"2026-08-25T11:21:35.932791726Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, dns-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-k9tccs with version 4.21.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005c61e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-k9tccs: failed waiting for cluster=\"np-version-upgrade-cluster-k9tccs\" in resourcegroup=\"rg-np-version-upgrade-k9tccs-dxkthfx2xcmq\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06b6d562-cc76-49e7-90e9-e0c893ebdaca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06b6d562-cc76-49e7-90e9-e0c893ebdaca\",\n      \"name\": \"06b6d562-cc76-49e7-90e9-e0c893ebdaca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:30.260205158Z\",\n      \"endTime\": \"2026-08-25T11:21:35.932791726Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, dns-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"620efc4df137449edf6e36bc5ee396e617e4c93be1f20afe74e9e360cbb49723","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"8535cc4178a95a0eda0b65492ec9ecf1ca1fb3239efd1f0b829eea75cb13b88d","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"a3f04b099b92e17296b69b5c0846c2aba1e01c878468735dffc3852678c7ee27","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0f70434aa7280dae0f02a3c793869c46e1b4bd7e7886836d4307001c25d29ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-27"],"prior_runs_affected":4,"prior_last_seen_at":"2026-07-29T14:33:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b2c400\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-v5fxchbb2prs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561d343f-9a10-4a71-ba6a-31ce4d8eda76\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561d343f-9a10-4a71-ba6a-31ce4d8eda76\",\n      \"name\": \"561d343f-9a10-4a71-ba6a-31ce4d8eda76\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:21:13.116412138Z\",\n      \"endTime\": \"2026-08-25T19:40:22.435169961Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b2c400\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-v5fxchbb2prs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561d343f-9a10-4a71-ba6a-31ce4d8eda76\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/561d343f-9a10-4a71-ba6a-31ce4d8eda76\",\n      \"name\": \"561d343f-9a10-4a71-ba6a-31ce4d8eda76\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:21:13.116412138Z\",\n      \"endTime\": \"2026-08-25T19:40:22.435169961Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b39a0\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-g89x2rmfntgq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ed45382-6139-4927-aeb6-b2580f808168\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ed45382-6139-4927-aeb6-b2580f808168\",\n      \"name\": \"8ed45382-6139-4927-aeb6-b2580f808168\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:07:09.765576241Z\",\n      \"endTime\": \"2026-08-25T11:26:17.763320122Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b39a0\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-g89x2rmfntgq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ed45382-6139-4927-aeb6-b2580f808168\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ed45382-6139-4927-aeb6-b2580f808168\",\n      \"name\": \"8ed45382-6139-4927-aeb6-b2580f808168\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:07:09.765576241Z\",\n      \"endTime\": \"2026-08-25T11:26:17.763320122Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001c189e0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-r5zg79zklshc\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/460dce50-0e02-4f7d-918e-303c53d250cd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/460dce50-0e02-4f7d-918e-303c53d250cd\",\n      \"name\": \"460dce50-0e02-4f7d-918e-303c53d250cd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:06.369521114Z\",\n      \"endTime\": \"2026-08-25T11:24:07.709199647Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001c189e0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-r5zg79zklshc\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/460dce50-0e02-4f7d-918e-303c53d250cd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/460dce50-0e02-4f7d-918e-303c53d250cd\",\n      \"name\": \"460dce50-0e02-4f7d-918e-303c53d250cd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:06.369521114Z\",\n      \"endTime\": \"2026-08-25T11:24:07.709199647Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"b76793c96776650aa6c2c9acf626989ef5411551675fc33c8a57f8cf74578d78","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"e709edfaf5822475a6c3cdc7e438cd913aefedf35c5c4aa2177cb9bc8689a712","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"f1d2bc77809969c73b592e28941870857aa223052d643ab35baa352ec051a614","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"91b64d02be224daeb152d864ec9bb135842f4f0c70f108c5bb00e21886c7eb97","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[0,0,0,3,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-21T16:22:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:299]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014739f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-54pbsl\" in resourcegroup=\"rg-hs-pre--hgwq5fppp7qz\" to finish deleting: GET https://rp.j6534144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03101c36-9e92-40e9-8333-009c7244042d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03101c36-9e92-40e9-8333-009c7244042d\",\n      \"name\": \"03101c36-9e92-40e9-8333-009c7244042d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T17:20:28.711507975Z\",\n      \"endTime\": \"2026-08-21T17:44:35.276549472Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:299]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014739f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-54pbsl\" in resourcegroup=\"rg-hs-pre--hgwq5fppp7qz\" to finish deleting: GET https://rp.j6534144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03101c36-9e92-40e9-8333-009c7244042d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03101c36-9e92-40e9-8333-009c7244042d\",\n      \"name\": \"03101c36-9e92-40e9-8333-009c7244042d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T17:20:28.711507975Z\",\n      \"endTime\": \"2026-08-21T17:44:35.276549472Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:299]: Unexpected error:\n    \u003c*errors.joinError | 0xc00136c168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7786mg2bm9fv\" to finish deleting: GET https://rp.j6534144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b466b32a-5f1d-4a5f-ba54-1dab632359ab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b466b32a-5f1d-4a5f-ba54-1dab632359ab\",\n      \"name\": \"b466b32a-5f1d-4a5f-ba54-1dab632359ab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T17:20:32.530035089Z\",\n      \"endTime\": \"2026-08-21T17:44:35.27735534Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:299]: Unexpected error:\n    \u003c*errors.joinError | 0xc00136c168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7786mg2bm9fv\" to finish deleting: GET https://rp.j6534144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b466b32a-5f1d-4a5f-ba54-1dab632359ab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b466b32a-5f1d-4a5f-ba54-1dab632359ab\",\n      \"name\": \"b466b32a-5f1d-4a5f-ba54-1dab632359ab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T17:20:32.530035089Z\",\n      \"endTime\": \"2026-08-21T17:44:35.27735534Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012240a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-5xdbvmxgwd5n\" to finish deleting: GET https://rp.j7397632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a782820-358c-4562-a28f-3ad7c668ca68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a782820-358c-4562-a28f-3ad7c668ca68\",\n      \"name\": \"8a782820-358c-4562-a28f-3ad7c668ca68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T04:23:24.977266828Z\",\n      \"endTime\": \"2026-08-21T04:47:26.492488581Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012240a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-5xdbvmxgwd5n\" to finish deleting: GET https://rp.j7397632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a782820-358c-4562-a28f-3ad7c668ca68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a782820-358c-4562-a28f-3ad7c668ca68\",\n      \"name\": \"8a782820-358c-4562-a28f-3ad7c668ca68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T04:23:24.977266828Z\",\n      \"endTime\": \"2026-08-21T04:47:26.492488581Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6647/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090836865526534144","occurred_at":"2026-08-21T16:22:00Z","signature_id":"0d1b73ed050b18456353cb4d2352218fe21c85c9eda609d938d0ed0a6d1dc131","pr_number":6647,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6647/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090836865526534144","occurred_at":"2026-08-21T16:22:00Z","signature_id":"67ec7c174fce546a47f303fa82220ae45719d138d9e2e4b62f431576840c8059","pr_number":6647,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090644128227397632","occurred_at":"2026-08-21T03:36:17Z","signature_id":"3db2ec536258c92a9174667711d55e05fd5d9a3ef77f875aa5ea0530b4cb8a7b","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"60a907245af237f1fcb597ff0d35bc01af7bd1e3900650496d648ae7c2abbb63","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [dns-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-vmkxjb\" with version 4.22.10 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001438440\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-vmkxjb: failed waiting for cluster=\"cluster-zstream-4-22-vmkxjb\" in resourcegroup=\"rg-zstream-upgrade-4-22-dgzc9xl48h57\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88ef8722-4c84-4313-a177-16c92648110e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88ef8722-4c84-4313-a177-16c92648110e\",\n      \"name\": \"88ef8722-4c84-4313-a177-16c92648110e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:49.211451249Z\",\n      \"endTime\": \"2026-08-25T11:21:55.960423286Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-vmkxjb\" with version 4.22.10 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001438440\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-vmkxjb: failed waiting for cluster=\"cluster-zstream-4-22-vmkxjb\" in resourcegroup=\"rg-zstream-upgrade-4-22-dgzc9xl48h57\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88ef8722-4c84-4313-a177-16c92648110e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88ef8722-4c84-4313-a177-16c92648110e\",\n      \"name\": \"88ef8722-4c84-4313-a177-16c92648110e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:49.211451249Z\",\n      \"endTime\": \"2026-08-25T11:21:55.960423286Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3cae0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-jfmt6ghkl4r6\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/276ea232-8f07-40f6-a841-fed05291e9a5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/276ea232-8f07-40f6-a841-fed05291e9a5\",\n      \"name\": \"276ea232-8f07-40f6-a841-fed05291e9a5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:29.221378186Z\",\n      \"endTime\": \"2026-08-25T11:21:35.933589807Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3cae0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-jfmt6ghkl4r6\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/276ea232-8f07-40f6-a841-fed05291e9a5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/276ea232-8f07-40f6-a841-fed05291e9a5\",\n      \"name\": \"276ea232-8f07-40f6-a841-fed05291e9a5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:29.221378186Z\",\n      \"endTime\": \"2026-08-25T11:21:35.933589807Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-w6h87f\" with version 4.20.34 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001936140\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-w6h87f: failed waiting for cluster=\"cluster-zstream-4-20-w6h87f\" in resourcegroup=\"rg-zstream-upgrade-4-20-xmx8p29fpbx4\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a563416d-3979-490a-9710-b74e5cae9107\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a563416d-3979-490a-9710-b74e5cae9107\",\n      \"name\": \"a563416d-3979-490a-9710-b74e5cae9107\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:29.203167213Z\",\n      \"endTime\": \"2026-08-25T11:21:35.888478672Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-w6h87f\" with version 4.20.34 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001936140\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-w6h87f: failed waiting for cluster=\"cluster-zstream-4-20-w6h87f\" in resourcegroup=\"rg-zstream-upgrade-4-20-xmx8p29fpbx4\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a563416d-3979-490a-9710-b74e5cae9107\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a563416d-3979-490a-9710-b74e5cae9107\",\n      \"name\": \"a563416d-3979-490a-9710-b74e5cae9107\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:29.203167213Z\",\n      \"endTime\": \"2026-08-25T11:21:35.888478672Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: dns-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"6e634d35cd843e9cfae5b6b27514de0cb7373b65ef36dac104cfb1c90125fcc9","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"a5d387bcc562b88ab6a23d38e6d2aa3de0eb793a989249e8ad448238a04782c1","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"ae1dbdff3202605f5db0ab0f7e502b028b65320bf6e7bd52d1ff183a02717a24","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"604364d97bb0ab0255e5c33341b7e0841ff36bd1d0c70f547552741d3f27bbc9","failure_pattern":"Unable to connect to the server: EOF)","search_query":"Unable to connect to the server: EOF)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,3,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T15:06:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS, private ingress, and private KeyVault and verify all are operational","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:152]: KAS should be reachable from VM via private endpoint (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nUnable to connect to the server: EOF)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:152]: KAS should be reachable from VM via private endpoint (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nUnable to connect to the server: EOF)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:196]: kubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nUnable to connect to the server: EOF)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:196]: kubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nUnable to connect to the server: EOF)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:165]: kubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nUnable to connect to the server: EOF)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:165]: kubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nUnable to connect to the server: EOF)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817798367350784","occurred_at":"2026-08-21T15:06:21Z","signature_id":"12740fe2d6d61cf441e4366b28f12d088ca342286578edc02db60262d6b170c1","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817798367350784","occurred_at":"2026-08-21T15:06:21Z","signature_id":"9e38ec5c20d23e8d209c2c2b4f32636885e8207d2981ec8540bd0b7c2a8e09e6","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817798367350784","occurred_at":"2026-08-21T15:06:21Z","signature_id":"9e38ec5c20d23e8d209c2c2b4f32636885e8207d2981ec8540bd0b7c2a8e09e6","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster fully-private, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20260630FromParam for cluster fully-private in resource group fully-private-grwst6zxhql9, error: failed waiting for cluster=\"fully-private\" in resourcegroup=\"fully-private-grwst6zxhql9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20260630FromParam for cluster fully-private in resource group fully-private-grwst6zxhql9, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":3,"after_last_push_count":0,"also_in":["PROD","STG"],"trend_counts":[0,0,3,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":35,"prior_last_seen_at":"2026-08-20T14:20:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS, private ingress, and private KeyVault and verify all are operational","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:100]: failed to create fully private HCP cluster \"fully-private\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0029fc600\u003e: \n    failed to create HCP cluster fully-private, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20260630FromParam for cluster fully-private in resource group fully-private-grwst6zxhql9, error: failed waiting for cluster=\"fully-private\" in resourcegroup=\"fully-private-grwst6zxhql9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20260630FromParam for cluster fully-private in resource group fully-private-grwst6zxhql9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:100]: failed to create fully private HCP cluster \"fully-private\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0029fc600\u003e: \n    failed to create HCP cluster fully-private, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20260630FromParam for cluster fully-private in resource group fully-private-grwst6zxhql9, error: failed waiting for cluster=\"fully-private\" in resourcegroup=\"fully-private-grwst6zxhql9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20260630FromParam for cluster fully-private in resource group fully-private-grwst6zxhql9, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:91]: failed to create private HCP cluster \"priv-np-scale\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002136a50\u003e: \n    failed to create HCP cluster priv-np-scale, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster priv-np-scale in resource group priv-np-scale-zkfmfd2hhlvt, error: failed waiting for cluster=\"priv-np-scale\" in resourcegroup=\"priv-np-scale-zkfmfd2hhlvt\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster priv-np-scale in resource group priv-np-scale-zkfmfd2hhlvt, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:91]: failed to create private HCP cluster \"priv-np-scale\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002136a50\u003e: \n    failed to create HCP cluster priv-np-scale, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster priv-np-scale in resource group priv-np-scale-zkfmfd2hhlvt, error: failed waiting for cluster=\"priv-np-scale\" in resourcegroup=\"priv-np-scale-zkfmfd2hhlvt\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster priv-np-scale in resource group priv-np-scale-zkfmfd2hhlvt, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:116]: failed to create fully private HCP cluster \"full-priv-kv\" with private KV\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001614e70\u003e: \n    failed waiting for cluster=\"full-priv-kv\" in resourcegroup=\"full-priv-kv-h8k8sz27jdqk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20260630AndWait for cluster full-priv-kv in resource group full-priv-kv-h8k8sz27jdqk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:116]: failed to create fully private HCP cluster \"full-priv-kv\" with private KV\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001614e70\u003e: \n    failed waiting for cluster=\"full-priv-kv\" in resourcegroup=\"full-priv-kv-h8k8sz27jdqk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20260630AndWait for cluster full-priv-kv in resource group full-priv-kv-h8k8sz27jdqk, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090443849187987456","occurred_at":"2026-08-20T14:20:26Z","signature_id":"030164acb30f2dbbf672de3a51088927223cf9230bbe93ac2bc5757d097d3729","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090443849187987456","occurred_at":"2026-08-20T14:20:26Z","signature_id":"5e749b5d7218678f3f8683b1e0d2510850d1213f30e492a7e9e0982cd6cdea23","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090443849187987456","occurred_at":"2026-08-20T14:20:26Z","signature_id":"e69e5a85d0e016d4801e1345cf988a8d0b41e00f41b331baea3bdb9477a0c546","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,1,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-19T08:55:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-tp6lxr8bfgv2\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/863038dc-ed8c-4013-be96-35e5dd9a5f59\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/863038dc-ed8c-4013-be96-35e5dd9a5f59\",\n  \"name\": \"863038dc-ed8c-4013-be96-35e5dd9a5f59\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-08-25T11:10:58.76533919Z\",\n  \"endTime\": \"2026-08-25T11:29:59.215293237Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-tp6lxr8bfgv2\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/863038dc-ed8c-4013-be96-35e5dd9a5f59\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/863038dc-ed8c-4013-be96-35e5dd9a5f59\",\n  \"name\": \"863038dc-ed8c-4013-be96-35e5dd9a5f59\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-08-25T11:10:58.76533919Z\",\n  \"endTime\": \"2026-08-25T11:29:59.215293237Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-update-nodes: failed waiting for nodepool=\"np-update-nodes\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-v9sgt7z9p57z\" to finish creating: GET https://rp.j8473600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d392562-a61e-4063-8716-f8bbe26726fb\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d392562-a61e-4063-8716-f8bbe26726fb\",\n  \"name\": \"0d392562-a61e-4063-8716-f8bbe26726fb\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-08-19T09:49:09.811303693Z\",\n  \"endTime\": \"2026-08-19T10:08:11.65820307Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-update-nodes: failed waiting for nodepool=\"np-update-nodes\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-v9sgt7z9p57z\" to finish creating: GET https://rp.j8473600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d392562-a61e-4063-8716-f8bbe26726fb\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d392562-a61e-4063-8716-f8bbe26726fb\",\n  \"name\": \"0d392562-a61e-4063-8716-f8bbe26726fb\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-08-19T09:49:09.811303693Z\",\n  \"endTime\": \"2026-08-19T10:08:11.65820307Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"f43c2f2c050759d71b643af6fc429ef7cf204b9a1f2b922e8824120be8851f15","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089999791768473600","occurred_at":"2026-08-19T08:55:55Z","signature_id":"2bedfe93590f3320ce1a1d2b5e550c02a75b1d68402e0b650ded60bc95e7e3d8","pr_number":6588,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,1,0,0,1,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-22T03:47:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-tpmsc2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9ab40\u003e: \n    DNS for route host agnhost-e2e-sample-app-vdv8k.apps.aro.r6l0b2y8t4z4p9n.rppd.j6613120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-vdv8k.apps.aro.r6l0b2y8t4z4p9n.rppd.j6613120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vdv8k.apps.aro.r6l0b2y8t4z4p9n.rppd.j6613120.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-tpmsc2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9ab40\u003e: \n    DNS for route host agnhost-e2e-sample-app-vdv8k.apps.aro.r6l0b2y8t4z4p9n.rppd.j6613120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-vdv8k.apps.aro.r6l0b2y8t4z4p9n.rppd.j6613120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vdv8k.apps.aro.r6l0b2y8t4z4p9n.rppd.j6613120.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:131]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b23c0\u003e: \n    DNS for route host agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:131]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b23c0\u003e: \n    DNS for route host agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091009379636613120","occurred_at":"2026-08-22T03:47:39Z","signature_id":"0522283d1fc1a443421be885233a74dcbe6c90c68314e2d70f47677b913d2059","pr_number":6607,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6600/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090054287706034176","occurred_at":"2026-08-19T12:32:25Z","signature_id":"3f9698e822bb6121785dfd4c9a4ed9d11389bae85f981bd4dacd92dce39fcb8e","pr_number":6600,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"2cc2a39959875f671b6207a8cb1637964bc140701e12d99f249891f56ec6e90a","failure_pattern":"ERROR CODE: DeploymentActive","search_query":"ERROR CODE: DeploymentActive","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=1)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-20T11:31:49Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=1)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/management/output (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-08-20T12:10:57.311Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1 description=\"Step output\\n  Kind: ARM\\n  Template: ./../dev-infrastructure/templates/output-mgmt.bicep\\n  Parameters: ./../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-20T12:10:57.943Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1\ntime=2026-08-20T12:10:58.421Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1 err=\"stamp 1: failed to run ARM step: failed to create deployment: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5367552-mgmt-1/providers/Microsoft.Resources/deployments/b33ca9cd40b2fdfdad4fc961f59230c1353c681efeada5d41f51b14954fa4cd6\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: DeploymentActive\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentActive\\\",\\n    \\\"message\\\": \\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5367552-mgmt-1/providers/Microsoft.Resources/deployments/b33ca9cd40b2fdfdad4fc961f59230c1353c681efeada5d41f51b14954fa4cd6' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '8/20/2026 12:10:57 PM' with correlationId '9748f509-1eae-448e-a55b-8ae0259f31d6', and will expire at '8/27/2026 12:10:57 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-19T01:54:26.847Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 description=\"Step output\\n  Kind: ARM\\n  Template: ../../dev-infrastructure/templates/output-mgmt.bicep\\n  Parameters: ../../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-19T01:54:27.623Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1\ntime=2026-08-19T01:54:28.068Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 err=\"stamp 1: failed to run ARM step: failed to create deployment: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9945216-mgmt-1/providers/Microsoft.Resources/deployments/21fa006528782108614538657b128ed438c46927450e49bcd34d09fae1d95b96\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: DeploymentActive\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentActive\\\",\\n    \\\"message\\\": \\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9945216-mgmt-1/providers/Microsoft.Resources/deployments/21fa006528782108614538657b128ed438c46927450e49bcd34d09fae1d95b96' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '8/19/2026 1:54:27 AM' with correlationId 'b1f79be8-ea5e-4ca3-9b19-2374cbb23eb8', and will expire at '8/26/2026 1:54:27 AM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090401415305367552","occurred_at":"2026-08-20T11:31:49Z","signature_id":"60f61d5cb7c890572bc4c320ced0f64c99e01d9ec051384d6144b65fd7b48f8a","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089884673239945216","occurred_at":"2026-08-19T01:18:28Z","signature_id":"dcf733b79532628fb81ebf58d69836a0011b2d8445bc977f5d9198084d7b8eca","pr_number":6588,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"56aad0b4262c7cedf5722ba39358ba8c22af49e4bbcaade93736c9c50ce6eb1a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-10"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-14T20:39:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00104e420\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-r77s4bbr26ht\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8c31bd00-7879-4578-bf29-c883b2f23367\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8c31bd00-7879-4578-bf29-c883b2f23367\",\n      \"name\": \"8c31bd00-7879-4578-bf29-c883b2f23367\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:19:33.22593953Z\",\n      \"endTime\": \"2026-08-25T19:38:33.945524152Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00104e420\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-r77s4bbr26ht\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8c31bd00-7879-4578-bf29-c883b2f23367\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8c31bd00-7879-4578-bf29-c883b2f23367\",\n      \"name\": \"8c31bd00-7879-4578-bf29-c883b2f23367\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:19:33.22593953Z\",\n      \"endTime\": \"2026-08-25T19:38:33.945524152Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa6640\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-97gvpsltv9sn\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/55ca21a5-264c-4b89-812c-1bd654f8f905\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/55ca21a5-264c-4b89-812c-1bd654f8f905\",\n      \"name\": \"55ca21a5-264c-4b89-812c-1bd654f8f905\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:24:51.992542762Z\",\n      \"endTime\": \"2026-08-25T14:43:53.990674833Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa6640\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-97gvpsltv9sn\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/55ca21a5-264c-4b89-812c-1bd654f8f905\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/55ca21a5-264c-4b89-812c-1bd654f8f905\",\n      \"name\": \"55ca21a5-264c-4b89-812c-1bd654f8f905\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:24:51.992542762Z\",\n      \"endTime\": \"2026-08-25T14:43:53.990674833Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"42b392a614867f2ce7fd49a1175ca961a846df1640ce64649fc51da9a67204d7","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"f04cb5f20e7c296508246409d131f2f25950ae8cf510b0c95d54eb7e1af46d29","pr_number":6641,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d188724e2ad8b2ff0009817ae8838a4d0cce4a292b1f33e9f843e12ff24e190c","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--8jnkhgqhx7vj/cluster-hs-pre-gcwzlp should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011ee120\u003e: \n    failed to create HCP cluster cluster-hs-pre-gcwzlp: failed waiting for cluster=\"cluster-hs-pre-gcwzlp\" in resourcegroup=\"rg-hs-pre--8jnkhgqhx7vj\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\",\n      \"name\": \"aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:18:58.14501483Z\",\n      \"endTime\": \"2026-08-25T19:38:02.43879336Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--8jnkhgqhx7vj/cluster-hs-pre-gcwzlp should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011ee120\u003e: \n    failed to create HCP cluster cluster-hs-pre-gcwzlp: failed waiting for cluster=\"cluster-hs-pre-gcwzlp\" in resourcegroup=\"rg-hs-pre--8jnkhgqhx7vj\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\",\n      \"name\": \"aacf9bd9-5b3a-49ff-8d49-6acfb8d7062d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:18:58.14501483Z\",\n      \"endTime\": \"2026-08-25T19:38:02.43879336Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ff8340\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-qhbwgqnksltm\" to finish creating: GET https://rp.j9625216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/549edf52-d404-4f5d-b33e-34f0219e1814\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/549edf52-d404-4f5d-b33e-34f0219e1814\",\n      \"name\": \"549edf52-d404-4f5d-b33e-34f0219e1814\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T15:41:53.637919079Z\",\n      \"endTime\": \"2026-08-25T16:00:58.269163772Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ff8340\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-qhbwgqnksltm\" to finish creating: GET https://rp.j9625216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/549edf52-d404-4f5d-b33e-34f0219e1814\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/549edf52-d404-4f5d-b33e-34f0219e1814\",\n      \"name\": \"549edf52-d404-4f5d-b33e-34f0219e1814\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T15:41:53.637919079Z\",\n      \"endTime\": \"2026-08-25T16:00:58.269163772Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"7a9b3449c6776fb39f3e7d85c2865848c6849a9fed0d320c413b71edcfe0b968","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092260279839625216","occurred_at":"2026-08-25T14:38:17Z","signature_id":"2b07939ee9bb49277e2d643f0c99927b263772f00f1ecc4f749207d5a2800b2e","pr_number":6648,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fc644a81a7601796e8c7e94724517f0056e9382ba0920b2aaa0529c449d6cba7","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, machine-approver, olm-operator, openshift-apiserver, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002200620\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-2pmrbkkbtqsr\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/27a59a4e-4c36-4607-a1b2-03e8280030f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/27a59a4e-4c36-4607-a1b2-03e8280030f6\",\n      \"name\": \"27a59a4e-4c36-4607-a1b2-03e8280030f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:13.29802062Z\",\n      \"endTime\": \"2026-08-25T14:42:13.632929999Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, dns-operator, cluster-version-operator, cluster-storage-operator, certified-operators-catalog, machine-approver, openshift-controller-manager, catalog-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, redhat-operators-catalog, openshift-apiserver, openshift-route-controller-manager, cluster-network-operator, cluster-node-tuning-operator, olm-operator, community-operators-catalog, cluster-policy-controller, hosted-cluster-config-operator, ingress-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002200620\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-2pmrbkkbtqsr\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/27a59a4e-4c36-4607-a1b2-03e8280030f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/27a59a4e-4c36-4607-a1b2-03e8280030f6\",\n      \"name\": \"27a59a4e-4c36-4607-a1b2-03e8280030f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:13.29802062Z\",\n      \"endTime\": \"2026-08-25T14:42:13.632929999Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, dns-operator, cluster-version-operator, cluster-storage-operator, certified-operators-catalog, machine-approver, openshift-controller-manager, catalog-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, redhat-operators-catalog, openshift-apiserver, openshift-route-controller-manager, cluster-network-operator, cluster-node-tuning-operator, olm-operator, community-operators-catalog, cluster-policy-controller, hosted-cluster-config-operator, ingress-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011ca240\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-d4mb4xht7542\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\",\n      \"name\": \"d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:29.491781901Z\",\n      \"endTime\": \"2026-08-25T14:42:33.748439526Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, cluster-node-tuning-operator, cluster-version-operator, cluster-policy-controller, olm-operator, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, redhat-marketplace-catalog, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, catalog-operator, machine-approver, openshift-route-controller-manager, hosted-cluster-config-operator, azure-cloud-controller-manager, packageserver, openshift-controller-manager, dns-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011ca240\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-d4mb4xht7542\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\",\n      \"name\": \"d1b31ca4-5e1a-4bef-8eae-3fae81f040ec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:29.491781901Z\",\n      \"endTime\": \"2026-08-25T14:42:33.748439526Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, cluster-node-tuning-operator, cluster-version-operator, cluster-policy-controller, olm-operator, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, redhat-marketplace-catalog, cluster-storage-operator, ingress-operator, csi-snapshot-controller-operator, catalog-operator, machine-approver, openshift-route-controller-manager, hosted-cluster-config-operator, azure-cloud-controller-manager, packageserver, openshift-controller-manager, dns-operator, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"51019fbbc87d86ea23801cb65819ccf80ec820166053874e8a6fcf0732993915","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"993ce51c1d7313554e2033361bc0c241e0479977c567c64410226f52de9d8006","pr_number":6641,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"de9ee73e1f42212ffe8bacf6081359c2c391b2c47c68ba07fa4cd11f00e7b015","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-20T17:48:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-vk6cfm\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001216240\u003e: \n    failed to create HCP cluster np-noedge-vk6cfm: failed waiting for cluster=\"np-noedge-vk6cfm\" in resourcegroup=\"rg-np-noedge-vk6cfm-r5z9ggrs4bzc\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d73055c-caa7-4581-b662-b31a5154ad44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d73055c-caa7-4581-b662-b31a5154ad44\",\n      \"name\": \"6d73055c-caa7-4581-b662-b31a5154ad44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:40.855489688Z\",\n      \"endTime\": \"2026-08-25T11:21:45.894005118Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, packageserver, olm-collect-profiles, community-operators-catalog, azure-cloud-controller-manager, cluster-storage-operator, konnectivity-agent, cluster-autoscaler, cluster-version-operator, csi-snapshot-controller-operator, catalog-operator, openshift-apiserver, cluster-image-registry-operator, certified-operators-catalog, capi-provider, openshift-route-controller-manager, cluster-policy-controller, hosted-cluster-config-operator, dns-operator, ignition-server, olm-operator, machine-approver, kube-controller-manager, kube-scheduler, cluster-network-operator, control-plane-pki-operator, openshift-controller-manager, redhat-operators-catalog, cluster-node-tuning-operator, ignition-server-proxy, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-vk6cfm\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001216240\u003e: \n    failed to create HCP cluster np-noedge-vk6cfm: failed waiting for cluster=\"np-noedge-vk6cfm\" in resourcegroup=\"rg-np-noedge-vk6cfm-r5z9ggrs4bzc\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d73055c-caa7-4581-b662-b31a5154ad44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d73055c-caa7-4581-b662-b31a5154ad44\",\n      \"name\": \"6d73055c-caa7-4581-b662-b31a5154ad44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:40.855489688Z\",\n      \"endTime\": \"2026-08-25T11:21:45.894005118Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, packageserver, olm-collect-profiles, community-operators-catalog, azure-cloud-controller-manager, cluster-storage-operator, konnectivity-agent, cluster-autoscaler, cluster-version-operator, csi-snapshot-controller-operator, catalog-operator, openshift-apiserver, cluster-image-registry-operator, certified-operators-catalog, capi-provider, openshift-route-controller-manager, cluster-policy-controller, hosted-cluster-config-operator, dns-operator, ignition-server, olm-operator, machine-approver, kube-controller-manager, kube-scheduler, cluster-network-operator, control-plane-pki-operator, openshift-controller-manager, redhat-operators-catalog, cluster-node-tuning-operator, ignition-server-proxy, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001314280\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-f7brrnlwp2l5\" to finish creating: GET https://rp.j8125696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1e6d169-deac-4e56-b97f-0a253becce60\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1e6d169-deac-4e56-b97f-0a253becce60\",\n      \"name\": \"e1e6d169-deac-4e56-b97f-0a253becce60\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T18:37:36.102317205Z\",\n      \"endTime\": \"2026-08-20T18:56:41.667600628Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, packageserver, redhat-operators-catalog, capi-provider, cluster-version-operator, cluster-storage-operator, dns-operator, olm-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-network-operator, ingress-operator, kube-scheduler, csi-snapshot-controller-operator, kube-controller-manager, openshift-route-controller-manager, cluster-image-registry-operator, olm-collect-profiles, community-operators-catalog, cluster-node-tuning-operator, redhat-marketplace-catalog, certified-operators-catalog, machine-approver, openshift-apiserver, azure-cloud-controller-manager, ignition-server-proxy, cluster-autoscaler, control-plane-pki-operator, openshift-controller-manager, konnectivity-agent, ignition-server; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001314280\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-f7brrnlwp2l5\" to finish creating: GET https://rp.j8125696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1e6d169-deac-4e56-b97f-0a253becce60\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1e6d169-deac-4e56-b97f-0a253becce60\",\n      \"name\": \"e1e6d169-deac-4e56-b97f-0a253becce60\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T18:37:36.102317205Z\",\n      \"endTime\": \"2026-08-20T18:56:41.667600628Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, packageserver, redhat-operators-catalog, capi-provider, cluster-version-operator, cluster-storage-operator, dns-operator, olm-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-network-operator, ingress-operator, kube-scheduler, csi-snapshot-controller-operator, kube-controller-manager, openshift-route-controller-manager, cluster-image-registry-operator, olm-collect-profiles, community-operators-catalog, cluster-node-tuning-operator, redhat-marketplace-catalog, certified-operators-catalog, machine-approver, openshift-apiserver, azure-cloud-controller-manager, ignition-server-proxy, cluster-autoscaler, control-plane-pki-operator, openshift-controller-manager, konnectivity-agent, ignition-server; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"e075d605863206fdc2ccadedfa2c3d789e15cebb06d5f64e9eeed7ecaf6fde31","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"d31df3c1d2280d7555d7fa65f94d10af826e62abc345c39568cdb055792e220d","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f135b58213daed6ea6fbc2013a314b004e48b63448926884614927be5a42fc0d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, cluster-network-operator, cluster-node-tuning-operator, dns-operator, hosted-cluster-config-operator, ingress-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8e400\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-mvgcw5tqmrmq\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8748ffe8-2b9e-4821-88fd-9115a64acf5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8748ffe8-2b9e-4821-88fd-9115a64acf5f\",\n      \"name\": \"8748ffe8-2b9e-4821-88fd-9115a64acf5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:19:00.213672088Z\",\n      \"endTime\": \"2026-08-25T19:38:02.372107389Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, olm-operator, ingress-operator, hosted-cluster-config-operator, dns-operator, packageserver, cluster-node-tuning-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8e400\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-mvgcw5tqmrmq\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8748ffe8-2b9e-4821-88fd-9115a64acf5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8748ffe8-2b9e-4821-88fd-9115a64acf5f\",\n      \"name\": \"8748ffe8-2b9e-4821-88fd-9115a64acf5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:19:00.213672088Z\",\n      \"endTime\": \"2026-08-25T19:38:02.372107389Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, olm-operator, ingress-operator, hosted-cluster-config-operator, dns-operator, packageserver, cluster-node-tuning-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ad6ee0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vb2vq2d7pgb7\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e043488-17a2-4487-83fe-b621f2697284\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e043488-17a2-4487-83fe-b621f2697284\",\n      \"name\": \"9e043488-17a2-4487-83fe-b621f2697284\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:38.82687874Z\",\n      \"endTime\": \"2026-08-25T11:22:45.901459654Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, hosted-cluster-config-operator, packageserver, cluster-node-tuning-operator, dns-operator, cluster-network-operator, ingress-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ad6ee0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vb2vq2d7pgb7\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e043488-17a2-4487-83fe-b621f2697284\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e043488-17a2-4487-83fe-b621f2697284\",\n      \"name\": \"9e043488-17a2-4487-83fe-b621f2697284\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:38.82687874Z\",\n      \"endTime\": \"2026-08-25T11:22:45.901459654Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, hosted-cluster-config-operator, packageserver, cluster-node-tuning-operator, dns-operator, cluster-network-operator, ingress-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"01abb2e6b898b7ccf9ba858e60ea62a7872111a48241f292195140bc087ead89","pr_number":6683,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"08d1216f17f57d3c84a42ade2f863024d6549f84bec76f38daa65fb0c0f271e4","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d6bc6e1fed29949cb7f899cfde9a499bb81c4498d10956c4f31ee3021f08a1b5","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-cgzvsfdzkmd6/cluster-candidate-4-20-gm7v89 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00146c980\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-gm7v89: failed waiting for cluster=\"cluster-candidate-4-20-gm7v89\" in resourcegroup=\"rg-candidate-4-20-cgzvsfdzkmd6\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/048ffc99-876d-41b4-8fb3-df2166c912e4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/048ffc99-876d-41b4-8fb3-df2166c912e4\",\n      \"name\": \"048ffc99-876d-41b4-8fb3-df2166c912e4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:19.870414032Z\",\n      \"endTime\": \"2026-08-25T14:42:23.711491128Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, ingress-operator, cluster-image-registry-operator, dns-operator, hosted-cluster-config-operator, packageserver, catalog-operator, cluster-version-operator, cluster-network-operator, csi-snapshot-controller-operator, cluster-policy-controller, cluster-node-tuning-operator, olm-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-storage-operator, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog, machine-approver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-cgzvsfdzkmd6/cluster-candidate-4-20-gm7v89 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00146c980\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-gm7v89: failed waiting for cluster=\"cluster-candidate-4-20-gm7v89\" in resourcegroup=\"rg-candidate-4-20-cgzvsfdzkmd6\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/048ffc99-876d-41b4-8fb3-df2166c912e4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/048ffc99-876d-41b4-8fb3-df2166c912e4\",\n      \"name\": \"048ffc99-876d-41b4-8fb3-df2166c912e4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:19.870414032Z\",\n      \"endTime\": \"2026-08-25T14:42:23.711491128Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, ingress-operator, cluster-image-registry-operator, dns-operator, hosted-cluster-config-operator, packageserver, catalog-operator, cluster-version-operator, cluster-network-operator, csi-snapshot-controller-operator, cluster-policy-controller, cluster-node-tuning-operator, olm-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-storage-operator, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog, machine-approver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c85320\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8h9ptxfrdc67\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ff2cef3-78a1-4805-a77d-d4ff614f91e4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ff2cef3-78a1-4805-a77d-d4ff614f91e4\",\n      \"name\": \"0ff2cef3-78a1-4805-a77d-d4ff614f91e4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:00:22.611774831Z\",\n      \"endTime\": \"2026-08-25T11:19:27.266054327Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, csi-snapshot-controller-operator, community-operators-catalog, certified-operators-catalog, catalog-operator, olm-operator, openshift-apiserver, cluster-policy-controller, dns-operator, openshift-route-controller-manager, ignition-server-proxy, packageserver, redhat-operators-catalog, control-plane-pki-operator, cluster-version-operator, cluster-network-operator, cluster-node-tuning-operator, konnectivity-agent, redhat-marketplace-catalog, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c85320\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8h9ptxfrdc67\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ff2cef3-78a1-4805-a77d-d4ff614f91e4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ff2cef3-78a1-4805-a77d-d4ff614f91e4\",\n      \"name\": \"0ff2cef3-78a1-4805-a77d-d4ff614f91e4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:00:22.611774831Z\",\n      \"endTime\": \"2026-08-25T11:19:27.266054327Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, csi-snapshot-controller-operator, community-operators-catalog, certified-operators-catalog, catalog-operator, olm-operator, openshift-apiserver, cluster-policy-controller, dns-operator, openshift-route-controller-manager, ignition-server-proxy, packageserver, redhat-operators-catalog, control-plane-pki-operator, cluster-version-operator, cluster-network-operator, cluster-node-tuning-operator, konnectivity-agent, redhat-marketplace-catalog, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"327150b988523757d3380ba7ee58ec97de40f9671944b2888f7d5817ed0ac98a","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"9f7d9da9375d115c3d27ec74ca79a5ddf3840aa4ba1cb9141a12954a39487c12","pr_number":6648,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1264ce1f65fb1c4e4b03ef0f0b493117b1b9becb99a5e03d914e5ae33ea84790","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-fl7ppg8w76k6/cluster-candidate-5-0-9h5fbf should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013b0300\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-9h5fbf: failed waiting for cluster=\"cluster-candidate-5-0-9h5fbf\" in resourcegroup=\"rg-candidate-5-0-fl7ppg8w76k6\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7b99ad-c6a9-48e9-9d4c-e880771fde30\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7b99ad-c6a9-48e9-9d4c-e880771fde30\",\n      \"name\": \"3e7b99ad-c6a9-48e9-9d4c-e880771fde30\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:20:49.919147854Z\",\n      \"endTime\": \"2026-08-25T14:39:53.656202045Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, packageserver, cluster-storage-operator, dns-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-fl7ppg8w76k6/cluster-candidate-5-0-9h5fbf should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013b0300\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-9h5fbf: failed waiting for cluster=\"cluster-candidate-5-0-9h5fbf\" in resourcegroup=\"rg-candidate-5-0-fl7ppg8w76k6\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7b99ad-c6a9-48e9-9d4c-e880771fde30\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7b99ad-c6a9-48e9-9d4c-e880771fde30\",\n      \"name\": \"3e7b99ad-c6a9-48e9-9d4c-e880771fde30\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:20:49.919147854Z\",\n      \"endTime\": \"2026-08-25T14:39:53.656202045Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, packageserver, cluster-storage-operator, dns-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e0b0e0\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-57kmhshxtcx2\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b802d9c9-7b78-4adb-ba82-1320188f9c6d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b802d9c9-7b78-4adb-ba82-1320188f9c6d\",\n      \"name\": \"b802d9c9-7b78-4adb-ba82-1320188f9c6d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:19.928553979Z\",\n      \"endTime\": \"2026-08-25T11:21:25.9465048Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, catalog-operator, cluster-storage-operator, dns-operator, ingress-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator, cluster-image-registry-operator, packageserver, olm-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e0b0e0\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-57kmhshxtcx2\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b802d9c9-7b78-4adb-ba82-1320188f9c6d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b802d9c9-7b78-4adb-ba82-1320188f9c6d\",\n      \"name\": \"b802d9c9-7b78-4adb-ba82-1320188f9c6d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:19.928553979Z\",\n      \"endTime\": \"2026-08-25T11:21:25.9465048Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, catalog-operator, cluster-storage-operator, dns-operator, ingress-operator, csi-snapshot-controller-operator, hosted-cluster-config-operator, cluster-image-registry-operator, packageserver, olm-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"3bb6f89d65a0e5459009870c86357e81a73e7068ce6cf13f3fa374106ca56368","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"01c47822900529458c20e60551800d6a17821b995d4851679b62a387c6efaa2c","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"52b927fb75e1a0e28c1c3ab5099caef3882ee435ade1e450f06515148c6a9d46","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-l7z8bjhz25gx/cluster-candidate-4-21-mj7vzs should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000690a00\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-mj7vzs: failed waiting for cluster=\"cluster-candidate-4-21-mj7vzs\" in resourcegroup=\"rg-candidate-4-21-l7z8bjhz25gx\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b46df85-5514-4dc0-92e8-a7d1f99eb864\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b46df85-5514-4dc0-92e8-a7d1f99eb864\",\n      \"name\": \"6b46df85-5514-4dc0-92e8-a7d1f99eb864\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:19.273281186Z\",\n      \"endTime\": \"2026-08-25T14:42:23.718787141Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-l7z8bjhz25gx/cluster-candidate-4-21-mj7vzs should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000690a00\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-mj7vzs: failed waiting for cluster=\"cluster-candidate-4-21-mj7vzs\" in resourcegroup=\"rg-candidate-4-21-l7z8bjhz25gx\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b46df85-5514-4dc0-92e8-a7d1f99eb864\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b46df85-5514-4dc0-92e8-a7d1f99eb864\",\n      \"name\": \"6b46df85-5514-4dc0-92e8-a7d1f99eb864\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:19.273281186Z\",\n      \"endTime\": \"2026-08-25T14:42:23.718787141Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b14940\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-56n9dv9dj6vs\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/877ebaca-f2ad-4e6f-9b78-d067abc099c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/877ebaca-f2ad-4e6f-9b78-d067abc099c8\",\n      \"name\": \"877ebaca-f2ad-4e6f-9b78-d067abc099c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:31.771881865Z\",\n      \"endTime\": \"2026-08-25T11:24:32.975906881Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, cluster-network-operator, certified-operators-catalog, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b14940\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-56n9dv9dj6vs\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/877ebaca-f2ad-4e6f-9b78-d067abc099c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/877ebaca-f2ad-4e6f-9b78-d067abc099c8\",\n      \"name\": \"877ebaca-f2ad-4e6f-9b78-d067abc099c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:31.771881865Z\",\n      \"endTime\": \"2026-08-25T11:24:32.975906881Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, cluster-network-operator, certified-operators-catalog, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"7d1767d3e893f8c308c1422c3d2aa396037e232ca87a62275c5cabd76581cee7","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"319f6c3fb2a06c2f17d4aa28d324ad963b4199781ff328a4f220cea67272ca73","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c975965241af31c57ec44c3fff1102f75c167bb738954ccc985f5a6bb7a81cd2","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator, dns-operator, ignition-server-proxy, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001100500\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-29j67455cd7q\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f324379d-3b99-49b4-a8dd-49ba6e3433d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f324379d-3b99-49b4-a8dd-49ba6e3433d2\",\n      \"name\": \"f324379d-3b99-49b4-a8dd-49ba6e3433d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:06.29234345Z\",\n      \"endTime\": \"2026-08-25T11:24:07.6554084Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, dns-operator, cluster-storage-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001100500\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-29j67455cd7q\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f324379d-3b99-49b4-a8dd-49ba6e3433d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f324379d-3b99-49b4-a8dd-49ba6e3433d2\",\n      \"name\": \"f324379d-3b99-49b4-a8dd-49ba6e3433d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:06.29234345Z\",\n      \"endTime\": \"2026-08-25T11:24:07.6554084Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, dns-operator, cluster-storage-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a8c220\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-c2x56bx2tvvb\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8707f304-4ff5-45ff-8354-0739ca14d80f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8707f304-4ff5-45ff-8354-0739ca14d80f\",\n      \"name\": \"8707f304-4ff5-45ff-8354-0739ca14d80f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:09.304723426Z\",\n      \"endTime\": \"2026-08-25T11:22:13.012989469Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, dns-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a8c220\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-c2x56bx2tvvb\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8707f304-4ff5-45ff-8354-0739ca14d80f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8707f304-4ff5-45ff-8354-0739ca14d80f\",\n      \"name\": \"8707f304-4ff5-45ff-8354-0739ca14d80f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:09.304723426Z\",\n      \"endTime\": \"2026-08-25T11:22:13.012989469Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, dns-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"2883b2d7f8a4d0400992958856aeb456b0ab19cc4ff95a4341bb724e485a3dc6","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"27c6b10bf1757b4f9a4d1318c60bd484d2401ca40c9413ba0f14effcfc7efe38","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f7ad14e1158d7d53f17395adc1367d8f6b3116098187de178cccafef487d5b29","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-22T10:03:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f5ae80\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-jpmgv9dccmdw\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ce80b32-21f8-4a00-a1a0-eb828a65f28d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ce80b32-21f8-4a00-a1a0-eb828a65f28d\",\n      \"name\": \"0ce80b32-21f8-4a00-a1a0-eb828a65f28d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:45.150172423Z\",\n      \"endTime\": \"2026-08-25T11:23:47.559518518Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f5ae80\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-jpmgv9dccmdw\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ce80b32-21f8-4a00-a1a0-eb828a65f28d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ce80b32-21f8-4a00-a1a0-eb828a65f28d\",\n      \"name\": \"0ce80b32-21f8-4a00-a1a0-eb828a65f28d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:45.150172423Z\",\n      \"endTime\": \"2026-08-25T11:23:47.559518518Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a5f0a0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-pp8z58bxmcb4\" to finish creating: GET https://rp.j3428992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2499d56f-998f-453a-9516-ce8ca4b2d871\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2499d56f-998f-453a-9516-ce8ca4b2d871\",\n      \"name\": \"2499d56f-998f-453a-9516-ce8ca4b2d871\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T10:46:33.945954582Z\",\n      \"endTime\": \"2026-08-22T11:05:38.525996595Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a5f0a0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-pp8z58bxmcb4\" to finish creating: GET https://rp.j3428992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2499d56f-998f-453a-9516-ce8ca4b2d871\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2499d56f-998f-453a-9516-ce8ca4b2d871\",\n      \"name\": \"2499d56f-998f-453a-9516-ce8ca4b2d871\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T10:46:33.945954582Z\",\n      \"endTime\": \"2026-08-22T11:05:38.525996595Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"7c0e987f55407504abfc17fba96a678154c2fb6eca07c78b46a987ce3627cfc1","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091103857353428992","occurred_at":"2026-08-22T10:03:05Z","signature_id":"c21db095d9a7b7ff0f65acbc17c905f119cc2e8936e61c276d6fb4034c19e49c","pr_number":6651,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2fc8daa06dcd0810c6da5b7d738c6be1c97e759c090dbe32d9900028e598f859","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-17T07:47:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-t7drrl95cd8z/cluster-candidate-4-20-t2lbd9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001a36920\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-t2lbd9: failed waiting for cluster=\"cluster-candidate-4-20-t2lbd9\" in resourcegroup=\"rg-candidate-4-20-t7drrl95cd8z\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/579deeb9-f336-4f04-8b2e-b7d680266538\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/579deeb9-f336-4f04-8b2e-b7d680266538\",\n      \"name\": \"579deeb9-f336-4f04-8b2e-b7d680266538\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:13:01.56047746Z\",\n      \"endTime\": \"2026-08-25T11:32:05.952606176Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-t7drrl95cd8z/cluster-candidate-4-20-t2lbd9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001a36920\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-t2lbd9: failed waiting for cluster=\"cluster-candidate-4-20-t2lbd9\" in resourcegroup=\"rg-candidate-4-20-t7drrl95cd8z\" to finish creating: GET https://rp.j0982912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/579deeb9-f336-4f04-8b2e-b7d680266538\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/579deeb9-f336-4f04-8b2e-b7d680266538\",\n      \"name\": \"579deeb9-f336-4f04-8b2e-b7d680266538\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:13:01.56047746Z\",\n      \"endTime\": \"2026-08-25T11:32:05.952606176Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-zhhnpcphpl7r/cluster-candidate-4-21-tcc8ch should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b2aa60\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-tcc8ch: failed waiting for cluster=\"cluster-candidate-4-21-tcc8ch\" in resourcegroup=\"rg-candidate-4-21-zhhnpcphpl7r\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06c23c7d-cba1-433b-881e-637949594df3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06c23c7d-cba1-433b-881e-637949594df3\",\n      \"name\": \"06c23c7d-cba1-433b-881e-637949594df3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:29.520227726Z\",\n      \"endTime\": \"2026-08-25T11:20:37.842143923Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-zhhnpcphpl7r/cluster-candidate-4-21-tcc8ch should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b2aa60\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-tcc8ch: failed waiting for cluster=\"cluster-candidate-4-21-tcc8ch\" in resourcegroup=\"rg-candidate-4-21-zhhnpcphpl7r\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06c23c7d-cba1-433b-881e-637949594df3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06c23c7d-cba1-433b-881e-637949594df3\",\n      \"name\": \"06c23c7d-cba1-433b-881e-637949594df3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:29.520227726Z\",\n      \"endTime\": \"2026-08-25T11:20:37.842143923Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092197655730982912","occurred_at":"2026-08-25T10:29:20Z","signature_id":"f2cb49f19339d4683d5c2bd5fd290c1a0f9cba1b7b1446ce110a22ee9a0d5a35","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"c23ab9a03bfcf50206e6f81c6de07ad7f3b1320c629158dce17b675a8a9b9d83","pr_number":6670,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2c8884cfc8ca38081890b0e85c74fa47ff1c59ad6753d36c6398a702d4e61b9d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,1,0,0,1,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-22T00:26:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010ec220\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-wgscgllb56rj\" to finish creating: GET https://rp.j7976064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/924dc541-9da5-4758-82ab-8fceb926a406\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/924dc541-9da5-4758-82ab-8fceb926a406\",\n      \"name\": \"924dc541-9da5-4758-82ab-8fceb926a406\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:17:11.361872214Z\",\n      \"endTime\": \"2026-08-22T01:36:20.474252514Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010ec220\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-wgscgllb56rj\" to finish creating: GET https://rp.j7976064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/924dc541-9da5-4758-82ab-8fceb926a406\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/924dc541-9da5-4758-82ab-8fceb926a406\",\n      \"name\": \"924dc541-9da5-4758-82ab-8fceb926a406\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:17:11.361872214Z\",\n      \"endTime\": \"2026-08-22T01:36:20.474252514Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005c4a00\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-xmgxlx2qvkgf\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/18aad82a-dc15-4cc0-bfaa-dde9546ca3b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/18aad82a-dc15-4cc0-bfaa-dde9546ca3b4\",\n      \"name\": \"18aad82a-dc15-4cc0-bfaa-dde9546ca3b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:30.596156848Z\",\n      \"endTime\": \"2026-08-19T19:47:39.777219011Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005c4a00\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-xmgxlx2qvkgf\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/18aad82a-dc15-4cc0-bfaa-dde9546ca3b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/18aad82a-dc15-4cc0-bfaa-dde9546ca3b4\",\n      \"name\": \"18aad82a-dc15-4cc0-bfaa-dde9546ca3b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:30.596156848Z\",\n      \"endTime\": \"2026-08-19T19:47:39.777219011Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090958673797976064","occurred_at":"2026-08-22T00:26:10Z","signature_id":"2b6787ba53692025280cc868873b273460045f3589a2601b36a19ada55076ff1","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"8b391d25577bbef6f7a5af7392059157b117715fdf88ddadc6f7148e824af69c","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"bd8944c9560147308a683885d199860a816c24f450fa26dcc2d571dd6c882239","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":0,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--sgwjdjgmhg59/cluster-hs-pre-28tpcf should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010fc840\u003e: \n    failed to create HCP cluster cluster-hs-pre-28tpcf: failed waiting for cluster=\"cluster-hs-pre-28tpcf\" in resourcegroup=\"rg-hs-pre--sgwjdjgmhg59\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd3092ff-ff71-4241-9dfd-bed54a05f55a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd3092ff-ff71-4241-9dfd-bed54a05f55a\",\n      \"name\": \"bd3092ff-ff71-4241-9dfd-bed54a05f55a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:23:47.826894918Z\",\n      \"endTime\": \"2026-08-25T16:42:48.770504975Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--sgwjdjgmhg59/cluster-hs-pre-28tpcf should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010fc840\u003e: \n    failed to create HCP cluster cluster-hs-pre-28tpcf: failed waiting for cluster=\"cluster-hs-pre-28tpcf\" in resourcegroup=\"rg-hs-pre--sgwjdjgmhg59\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd3092ff-ff71-4241-9dfd-bed54a05f55a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd3092ff-ff71-4241-9dfd-bed54a05f55a\",\n      \"name\": \"bd3092ff-ff71-4241-9dfd-bed54a05f55a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:23:47.826894918Z\",\n      \"endTime\": \"2026-08-25T16:42:48.770504975Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:93]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a6cce0\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-pcmvmmpvpflg\" to finish creating: GET https://rp.j9594112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52000f23-e3aa-465f-b2e6-2408f772e24f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52000f23-e3aa-465f-b2e6-2408f772e24f\",\n      \"name\": \"52000f23-e3aa-465f-b2e6-2408f772e24f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:59:13.55636864Z\",\n      \"endTime\": \"2026-08-25T12:18:21.188692138Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:93]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a6cce0\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-pcmvmmpvpflg\" to finish creating: GET https://rp.j9594112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52000f23-e3aa-465f-b2e6-2408f772e24f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52000f23-e3aa-465f-b2e6-2408f772e24f\",\n      \"name\": \"52000f23-e3aa-465f-b2e6-2408f772e24f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:59:13.55636864Z\",\n      \"endTime\": \"2026-08-25T12:18:21.188692138Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269882870599680","occurred_at":"2026-08-25T15:16:27Z","signature_id":"b94b017a26d7c85ad3a642438367f43e24dfb07c60186268412be86721621933","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092208871509594112","occurred_at":"2026-08-25T11:14:00Z","signature_id":"85a73bf3207678bf07196fd0dce16f63b867de9d5fd5b8dcc20aa62ad1433a40","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fe6b01560ac49646fd0a46c48f672cf2fc0ebbf844ea4576bb0cab7a46df7ce8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":0,"also_in":["PROD"],"trend_counts":[0,0,0,0,1,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-10","2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-22T10:03:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-cbhfzqtvb9th/cluster-candidate-4-21-5bbfxw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea2680\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-5bbfxw: failed waiting for cluster=\"cluster-candidate-4-21-5bbfxw\" in resourcegroup=\"rg-candidate-4-21-cbhfzqtvb9th\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n      \"name\": \"a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T17:28:32.844651935Z\",\n      \"endTime\": \"2026-08-24T17:47:34.560561952Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-cbhfzqtvb9th/cluster-candidate-4-21-5bbfxw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea2680\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-5bbfxw: failed waiting for cluster=\"cluster-candidate-4-21-5bbfxw\" in resourcegroup=\"rg-candidate-4-21-cbhfzqtvb9th\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n      \"name\": \"a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T17:28:32.844651935Z\",\n      \"endTime\": \"2026-08-24T17:47:34.560561952Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00099a320\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-vz6d75d4vrkm\" to finish creating: GET https://rp.j3428992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07b48c49-439f-4161-825c-a3f7e1694990\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07b48c49-439f-4161-825c-a3f7e1694990\",\n      \"name\": \"07b48c49-439f-4161-825c-a3f7e1694990\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T10:45:13.181167652Z\",\n      \"endTime\": \"2026-08-22T11:04:18.393945445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00099a320\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-vz6d75d4vrkm\" to finish creating: GET https://rp.j3428992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07b48c49-439f-4161-825c-a3f7e1694990\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07b48c49-439f-4161-825c-a3f7e1694990\",\n      \"name\": \"07b48c49-439f-4161-825c-a3f7e1694990\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T10:45:13.181167652Z\",\n      \"endTime\": \"2026-08-22T11:04:18.393945445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"94e14a4948dabf46772c6e049c6c316329369f07279d585a70084b3a5b07d8b8","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091103857353428992","occurred_at":"2026-08-22T10:03:05Z","signature_id":"c74ff041689685f9d2893c509b1ffbe4205a6aa02762660611f9c1e7f98334d4","pr_number":6651,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"03191c7ee6ef10f05815abde7f8b6f6bad2b4b3b1acdea0c834f5ddce823bc91","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: externalAuths, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":3,"also_in":["STG"],"trend_counts":[2,1,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-20T23:23:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012201b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-fnngq6qtshfw\" to finish deleting: GET https://rp.j8029568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26314739-180a-45be-9bc0-74aedd9378b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26314739-180a-45be-9bc0-74aedd9378b7\",\n      \"name\": \"26314739-180a-45be-9bc0-74aedd9378b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T00:42:44.968602571Z\",\n      \"endTime\": \"2026-08-21T01:06:45.158393217Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sakjl0nbdbh3dlb0ni55h84f7c279gs still exists (deletion dispatched at 2026-08-21T00:42:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012201b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-fnngq6qtshfw\" to finish deleting: GET https://rp.j8029568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26314739-180a-45be-9bc0-74aedd9378b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26314739-180a-45be-9bc0-74aedd9378b7\",\n      \"name\": \"26314739-180a-45be-9bc0-74aedd9378b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T00:42:44.968602571Z\",\n      \"endTime\": \"2026-08-21T01:06:45.158393217Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sakjl0nbdbh3dlb0ni55h84f7c279gs still exists (deletion dispatched at 2026-08-21T00:42:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012ae180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-jflf6brl2gcq\" to finish deleting: GET https://rp.j2645888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0d2b3f4-a9c9-4fe3-9d6c-919fb891cf66\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0d2b3f4-a9c9-4fe3-9d6c-919fb891cf66\",\n      \"name\": \"b0d2b3f4-a9c9-4fe3-9d6c-919fb891cf66\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:41:11.57390019Z\",\n      \"endTime\": \"2026-08-19T16:05:19.863368819Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ne21gf9tvn49fdnint8mmarsqg6fl still exists (deletion dispatched at 2026-08-19T15:41:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012ae180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-jflf6brl2gcq\" to finish deleting: GET https://rp.j2645888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0d2b3f4-a9c9-4fe3-9d6c-919fb891cf66\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0d2b3f4-a9c9-4fe3-9d6c-919fb891cf66\",\n      \"name\": \"b0d2b3f4-a9c9-4fe3-9d6c-919fb891cf66\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:41:11.57390019Z\",\n      \"endTime\": \"2026-08-19T16:05:19.863368819Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ne21gf9tvn49fdnint8mmarsqg6fl still exists (deletion dispatched at 2026-08-19T15:41:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090580629858029568","occurred_at":"2026-08-20T23:23:55Z","signature_id":"a688387e283d257bce8ddbe41e0174e2034ceb6121049d31c33a73bba2fccaa9","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"e22cca04f92354e3143765d7692aaa7085730c8c9dec1035cae8159d9630b31b","pr_number":6601,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"3663019c95fe16313f2b4252470b99823cc8609903410909837b48e27b5716a2","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": no upgrade path to update channel \"\u003cocp-channel\u003e\" is currently available for this cluster; a channel appears in the cluster's desired version channels only when an upgrade edge to a release in that channel exists; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-21T10:34:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:185]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-jxhbpr\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000827a10\u003e: \n    PATCH https://rp.j1906688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-nk4xz5qj6lkf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-jxhbpr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": no upgrade path to update channel \\\"candidate-4.22\\\" is currently available for this cluster; a channel appears in the cluster's desired version channels only when an upgrade edge to a release in that channel exists\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:185]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-jxhbpr\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000827a10\u003e: \n    PATCH https://rp.j1906688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-nk4xz5qj6lkf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-jxhbpr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": no upgrade path to update channel \\\"candidate-4.22\\\" is currently available for this cluster; a channel appears in the cluster's desired version channels only when an upgrade edge to a release in that channel exists\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:185]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-gq8xsp\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008705a0\u003e: \n    PATCH https://rp.j4357760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-djzz4pj5vnt8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-gq8xsp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": no upgrade path to update channel \\\"candidate-4.22\\\" is currently available for this cluster; a channel appears in the cluster's desired version channels only when an upgrade edge to a release in that channel exists\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:185]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-gq8xsp\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008705a0\u003e: \n    PATCH https://rp.j4357760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-djzz4pj5vnt8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-gq8xsp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": no upgrade path to update channel \\\"candidate-4.22\\\" is currently available for this cluster; a channel appears in the cluster's desired version channels only when an upgrade edge to a release in that channel exists\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6644/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090749265121906688","occurred_at":"2026-08-21T10:34:03Z","signature_id":"3ed03a19f79f0b1e2f0a12412d179fee0ef3e85a880c1f848954a744c9dbd32b","pr_number":6644,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090748107674357760","occurred_at":"2026-08-21T10:27:08Z","signature_id":"dd42386bd958f86f2faaef3c620305cbb99e3c2d8f582189054329bbccddbac1","pr_number":6612,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d6e88ce654a1e44ba45ba96ba70b0d0713c8f4a869b2a72dfffa4d3817a99e4e","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to get pod logs: an error on the server (\"unknown\") has prevented the request from succeeding (get pod \u003cpod\u003e)","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node i3f1v3w5b2n5j4l-np-1-ld9tc-cz8dm: failed to get pod logs: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-s5lz7)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-20T17:21:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:142]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc00163c708\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node i3f1v3w5b2n5j4l-np-1-ld9tc-cz8dm: failed to get pod logs: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-s5lz7)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:142]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc00163c708\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node i3f1v3w5b2n5j4l-np-1-ld9tc-cz8dm: failed to get pod logs: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-s5lz7)\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:142]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc001412c18\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node u9c4k8b0z1p8s9a-np-1-hhdmj-d5z7z: failed to get pod logs: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-8cndt)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:142]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc001412c18\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node u9c4k8b0z1p8s9a-np-1-hhdmj-d5z7z: failed to get pod logs: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-8cndt)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090489518846119936","occurred_at":"2026-08-20T17:21:54Z","signature_id":"676940ccfda45dd73c139c03c60293cf2f01a67e1fa939f78d3538b86b1422a0","pr_number":6278,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090439005190492160","occurred_at":"2026-08-20T14:00:59Z","signature_id":"0ba8df1b3d711fd1d88f28d00cded9c01dc9d64544970f2a555dd4f85fade54f","pr_number":6584,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1c7abf95f6b325e2b7136a3af3c492c951894fb6cb7083b18c5b7aad3165e0e0","failure_pattern":"alert [hcp] KubeContainerWaiting fired","search_query":"alert [hcp] KubeContainerWaiting fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeContainerWaiting does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":8,"trend_counts":[4,2,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":40,"prior_last_seen_at":"2026-08-19T18:19:42Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeContainerWaiting does not fire","occurrences":2}],"full_error_samples":["alert fired 8 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T20:14:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1797504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9r1sjk7nl7cq2gcvnm7d10nbm2ae5f-i2j7w2x8e1s9q5g\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9r1sjk7nl7cq2gcvnm7d10nbm2ae5f-i2j7w2x8e1s9q5g on container healthz has been in waiting state for longer than 1 hour.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T20:14:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1797504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9r1sjk7nl7cq2gcvnm7d10nbm2ae5f-i2j7w2x8e1s9q5g\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9r1sjk7nl7cq2gcvnm7d10nbm2ae5f-i2j7w2x8e1s9q5g on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T20:14:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1797504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9r1sjk7nl7cq2gcvnm7d10nbm2ae5f-i2j7w2x8e1s9q5g\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9r1sjk7nl7cq2gcvnm7d10nbm2ae5f-i2j7w2x8e1s9q5g on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T20:14:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1797504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9r1sjk7nl7cq2gcvnm7d10nbm2ae5f-i2j7w2x8e1s9q5g\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9r1sjk7nl7cq2gcvnm7d10nbm2ae5f-i2j7w2x8e1s9q5g on container etcd has been in waiting state for longer than 1 hour.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T20:15:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1797504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9r2ec2tegi43jr5a45h7ojphgg4rhv-f0v3l6y2u5x0q0u\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9r2ec2tegi43jr5a45h7ojphgg4rhv-f0v3l6y2u5x0q0u on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T20:15:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1797504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9r2ec2tegi43jr5a45h7ojphgg4rhv-f0v3l6y2u5x0q0u\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9r2ec2tegi43jr5a45h7ojphgg4rhv-f0v3l6y2u5x0q0u on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T20:15:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1797504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9r2ec2tegi43jr5a45h7ojphgg4rhv-f0v3l6y2u5x0q0u\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9r2ec2tegi43jr5a45h7ojphgg4rhv-f0v3l6y2u5x0q0u on container etcd has been in waiting state for longer than 1 hour.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T20:15:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1797504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9r2ec2tegi43jr5a45h7ojphgg4rhv-f0v3l6y2u5x0q0u\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9r2ec2tegi43jr5a45h7ojphgg4rhv-f0v3l6y2u5x0q0u on container healthz has been in waiting state for longer than 1 hour.","alert fired 45 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:55:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"manager\", namespace=\"ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q\", pod=\"capi-provider-58999f64dc-bltc4\", severity=\"warning\"\n  Description: pod/capi-provider-58999f64dc-bltc4 in namespace ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q on container manager has been in waiting state for longer than 1 hour.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:56:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h on container etcd has been in waiting state for longer than 1 hour.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:56:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:56:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q\", pod=\"etcd-0\", severity=\"warning\"\n  Description: pod/etcd-0 in namespace ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q on container etcd has been in waiting state for longer than 1 hour.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:56:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:56:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q\", pod=\"etcd-0\", severity=\"warning\"\n  Description: pod/etcd-0 in namespace ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:56:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q\", pod=\"etcd-0\", severity=\"warning\"\n  Description: pod/etcd-0 in namespace ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q on container healthz has been in waiting state for longer than 1 hour.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:56:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q\", pod=\"etcd-0\", severity=\"warning\"\n  Description: pod/etcd-0 in namespace ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:56:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h on container healthz has been in waiting state for longer than 1 hour.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e on container healthz has been in waiting state for longer than 1 hour.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f on container etcd has been in waiting state for longer than 1 hour.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster on container etcd has been in waiting state for longer than 1 hour.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e on container etcd has been in waiting state for longer than 1 hour.\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f on container healthz has been in waiting state for longer than 1 hour.\nFiring 17:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 18:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster on container healthz has been in waiting state for longer than 1 hour.\nFiring 19:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s on container healthz has been in waiting state for longer than 1 hour.\nFiring 20:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 21:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s on container etcd has been in waiting state for longer than 1 hour.\nFiring 22:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 23:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 24:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 25:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 26:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:58:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 27:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:58:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d on container etcd has been in waiting state for longer than 1 hour.\nFiring 28:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:58:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d on container healthz has been in waiting state for longer than 1 hour.\nFiring 29:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:58:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r on container etcd has been in waiting state for longer than 1 hour.\nFiring 30:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:58:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 31:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:58:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r on container healthz has been in waiting state for longer than 1 hour.\nFiring 32:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:58:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 33:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:58:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 34:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:59:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress on container healthz has been in waiting state for longer than 1 hour.\nFiring 35:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:59:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress on container etcd has been in waiting state for longer than 1 hour.\nFiring 36:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:59:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 37:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:59:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 38:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:01:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 39:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:01:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 40:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:01:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster on container healthz has been in waiting state for longer than 1 hour.\nFiring 41:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:01:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 42:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:01:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster on container etcd has been in waiting state for longer than 1 hour.\nFiring 43:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:01:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster on container etcd has been in waiting state for longer than 1 hour.\nFiring 44:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:01:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 45:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:01:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster on container healthz has been in waiting state for longer than 1 hour."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"015cf07ce6f663064e0b25d73a7d2265851a63a80888900249935eba2609bfae","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"638796d67faa614987f4d56ac4f93565ff06ae312263ef561431a0eaf8bad2d8","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":8,"trend_counts":[1,0,0,2,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":91,"prior_last_seen_at":"2026-08-21T20:12:06Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T20:56:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3460096-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"maestro\", pod=\"maestro-agent-666f6c69f9-jfc5v\", severity=\"warning\"\n  Description: Pod maestro/maestro-agent-666f6c69f9-jfc5v has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T02:42:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6439552-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-7c5f797746-w2wz8\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-7c5f797746-w2wz8 has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894760083460096","occurred_at":"2026-08-21T20:12:06Z","signature_id":"271bd166fd209036f953ac22fa2a76816c81cef505fa3146bce1d1786a3f3845","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"9855d4e08bca12aa1036cc00d95206a600dd2efa6c513fbdd3e506c0bb255560","pr_number":6617,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"470aab7d7a3605e8633a637a70d37da16684c6630946576af6d433f851290640","failure_pattern":"alert [svc] UJClusterProvisionErrors1h5m fired","search_query":"alert [svc] UJClusterProvisionErrors1h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors1h5m does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-27"],"prior_runs_affected":18,"prior_last_seen_at":"2026-07-31T12:10:07Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors1h5m does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T11:31:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"UJClusterProvisionErrors1h5m\", cluster=\"ci01-j2121984-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2121984/providers/microsoft.monitor/accounts/services-j2121984\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n  Description: More than 72% of cluster create (install) operations are in failed state, indicating a fast error budget burn (14.4x) that would exhaust the 95% SLO budget in ~12 hours. A regional install failure of this magnitude typically points at a shared dependency (e.g. registry, DNS, or ARM) rather than individual clusters.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-25T11:29:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"UJClusterProvisionErrors1h5m\", cluster=\"ci01-j5488384-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5488384/providers/microsoft.monitor/accounts/services-j5488384\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n  Description: More than 72% of cluster create (install) operations are in failed state, indicating a fast error budget burn (14.4x) that would exhaust the 95% SLO budget in ~12 hours. A regional install failure of this magnitude typically points at a shared dependency (e.g. registry, DNS, or ARM) rather than individual clusters."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"8726d474691b88bafaa41f02f06ab47cee7673893f1cb6c5f94125da19bae517","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"7ed12c6bca5e7dc43cf2acfe3e266ecf9c33df65853c31ec85ae02d6ffec33ff","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-20T23:44:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6669/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092147015625478144","occurred_at":"2026-08-25T07:08:13Z","signature_id":"ee111baab05f6b8ee26f353e865437e1bffda7140d9fc154398f141cd776e6f7","pr_number":6669,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090585791523721216","occurred_at":"2026-08-20T23:44:28Z","signature_id":"ee111baab05f6b8ee26f353e865437e1bffda7140d9fc154398f141cd776e6f7","pr_number":6584,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"784380ace55b355f47065469da48bc1c7b73547d879efb7edb5ed96de913234e","failure_pattern":"couldn't get current server API group list: the server has asked for the client to provide credentials","search_query":"couldn't get current server API group list: the server has asked for the client to provide credentials","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[1,0,1,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":11,"prior_last_seen_at":"2026-08-21T01:30:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:347]: Timed out after 300.004s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:332 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0xc000c401b0\u003e: \n    E0821 02:46:28.623242    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0821 02:46:28.657456    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0821 02:46:28.675878    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0821 02:46:28.697350    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0821 02:46:28.710066    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    error: You must be logged in to the server (the server has asked for the client to provide credentials)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:347]: Timed out after 300.004s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:332 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0xc000c401b0\u003e: \n    E0821 02:46:28.623242    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0821 02:46:28.657456    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0821 02:46:28.675878    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0821 02:46:28.697350    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0821 02:46:28.710066    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    error: You must be logged in to the server (the server has asked for the client to provide credentials)\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:347]: Timed out after 300.001s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:332 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0xc001943670\u003e: \n    E0820 10:14:05.864231    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0820 10:14:05.892161    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0820 10:14:05.910498    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0820 10:14:05.930980    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0820 10:14:05.946544    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    error: You must be logged in to the server (the server has asked for the client to provide credentials)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:347]: Timed out after 300.001s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:332 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0xc001943670\u003e: \n    E0820 10:14:05.864231    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0820 10:14:05.892161    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0820 10:14:05.910498    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0820 10:14:05.930980    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0820 10:14:05.946544    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    error: You must be logged in to the server (the server has asked for the client to provide credentials)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090612430223708160","occurred_at":"2026-08-21T01:30:11Z","signature_id":"f1894cbb696f520fee9ff4522a3cde239d81bc377f24ec4767de138d2980be5a","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"6eb333897f7f5d34b2448148e7185a4034de5809364747874cf76879c5e07670","pr_number":6583,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"17e5892e8c32879055400ab93a946ce79ad6cc684588825e1e1a488695d06f44","failure_pattern":"failed to poll cluster \"\u003ccluster\u003e\" authorized CIDRs update to completion (timeout '10.000000' minutes)","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,1,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:407]: failed to poll cluster \"cidr-connectivity-test\" authorized CIDRs update to completion (timeout '10.000000' minutes)\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:407]: failed to poll cluster \"cidr-connectivity-test\" authorized CIDRs update to completion (timeout '10.000000' minutes)\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"e903e6214fc854d426efd67d556189c90371a5d9fd33b4c1bbc8c31c92089a9a","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"e903e6214fc854d426efd67d556189c90371a5d9fd33b4c1bbc8c31c92089a9a","pr_number":6519,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"911f609de0f6334698a9ae03ee21b22f86f10fb03ad2176871cac7bbc4b43a5f","failure_pattern":"failed to select node pool \u003cnodepool\u003e version for \u003cocp-channel\u003e channel: Get \"\u003curl\u003e\": dial tcp: lookup api","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-20T16:56:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.208.156:42093-\u003e172.30.0.10:53: i/o timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.208.156:42093-\u003e172.30.0.10:53: i/o timeout","fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.131.120.38:40027-\u003e172.30.0.10:53: i/o timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.131.120.38:40027-\u003e172.30.0.10:53: i/o timeout"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","signature_id":"7171e132341694fb10089af1503bd1696a386e558f7ad492e46e056500959d3b","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6620/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090483026692673536","occurred_at":"2026-08-20T16:56:05Z","signature_id":"2d5072c48a74af035eca1d7e78f0a0c529eca7b4bf1560e30dd4259ba323f657","pr_number":6620,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bff39f67862ef72eccdd3c2006d48756f417856635339107ace9fde9b0a679a4","failure_pattern":"stamp 2 scheduling must have requested resources","search_query":"stamp 2 scheduling must have requested resources","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have scheduling data for ready management clusters","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.31545741324921134,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-22T10:03:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have scheduling data for ready management clusters","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.001s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:155 with:\nstamp 2 scheduling must have requested resources\nExpected\n    \u003cv1.ResourceList | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.001s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:155 with:\nstamp 2 scheduling must have requested resources\nExpected\n    \u003cv1.ResourceList | len:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"b261dd52f676523d8a989fb0761bdfb6871af15b6d8ed758e7d327e30440bdf8","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091103857353428992","occurred_at":"2026-08-22T10:03:05Z","signature_id":"b261dd52f676523d8a989fb0761bdfb6871af15b6d8ed758e7d327e30440bdf8","pr_number":6651,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6200cb8cea81a44cbffdcbf30f49e1b2e8791793e7e07d52640fca8ceb6c92c0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, machine-approver, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ce01c0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-vm8pwb95swzd\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\",\n      \"name\": \"17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:11.32352815Z\",\n      \"endTime\": \"2026-08-25T11:22:15.832343372Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, cluster-image-registry-operator, dns-operator, kube-storage-version-migrator, cluster-policy-controller, hosted-cluster-config-operator, cluster-storage-operator, catalog-operator, community-operators-catalog, machine-approver, openshift-route-controller-manager, cluster-network-operator, csi-snapshot-controller-operator, packageserver, certified-operators-catalog, openshift-controller-manager, cluster-version-operator, redhat-operators-catalog, openshift-apiserver, ingress-operator, olm-operator, azure-cloud-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ce01c0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-vm8pwb95swzd\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\",\n      \"name\": \"17b6acf0-f165-4ca6-b8c3-e2f36eb085c4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:11.32352815Z\",\n      \"endTime\": \"2026-08-25T11:22:15.832343372Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, cluster-image-registry-operator, dns-operator, kube-storage-version-migrator, cluster-policy-controller, hosted-cluster-config-operator, cluster-storage-operator, catalog-operator, community-operators-catalog, machine-approver, openshift-route-controller-manager, cluster-network-operator, csi-snapshot-controller-operator, packageserver, certified-operators-catalog, openshift-controller-manager, cluster-version-operator, redhat-operators-catalog, openshift-apiserver, ingress-operator, olm-operator, azure-cloud-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-bxvq84\" with version 4.21.29 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a00480\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-bxvq84: failed waiting for cluster=\"cluster-zstream-4-21-bxvq84\" in resourcegroup=\"rg-zstream-upgrade-4-21-f772hgcp6g6p\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c10c2eef-8fa9-4a27-b95a-35876b3db418\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c10c2eef-8fa9-4a27-b95a-35876b3db418\",\n      \"name\": \"c10c2eef-8fa9-4a27-b95a-35876b3db418\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:29.100638601Z\",\n      \"endTime\": \"2026-08-25T11:21:36.009069993Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, cluster-image-registry-operator, catalog-operator, ingress-operator, csi-snapshot-controller-operator, packageserver, machine-approver, azure-cloud-controller-manager, cluster-network-operator, redhat-marketplace-catalog, cluster-node-tuning-operator, hosted-cluster-config-operator, dns-operator, cluster-policy-controller, redhat-operators-catalog, cluster-version-operator, openshift-controller-manager, olm-operator, community-operators-catalog, openshift-route-controller-manager, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-bxvq84\" with version 4.21.29 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a00480\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-bxvq84: failed waiting for cluster=\"cluster-zstream-4-21-bxvq84\" in resourcegroup=\"rg-zstream-upgrade-4-21-f772hgcp6g6p\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c10c2eef-8fa9-4a27-b95a-35876b3db418\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c10c2eef-8fa9-4a27-b95a-35876b3db418\",\n      \"name\": \"c10c2eef-8fa9-4a27-b95a-35876b3db418\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:29.100638601Z\",\n      \"endTime\": \"2026-08-25T11:21:36.009069993Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, cluster-image-registry-operator, catalog-operator, ingress-operator, csi-snapshot-controller-operator, packageserver, machine-approver, azure-cloud-controller-manager, cluster-network-operator, redhat-marketplace-catalog, cluster-node-tuning-operator, hosted-cluster-config-operator, dns-operator, cluster-policy-controller, redhat-operators-catalog, cluster-version-operator, openshift-controller-manager, olm-operator, community-operators-catalog, openshift-route-controller-manager, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"84e02a1a3635ca3e59119a735efac8221692eec41952c2200fb85b25108b853d","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"fb329634cdbcd47b225aa1460092969d8489ffd2ea14b20506198098253816f6","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"efea2dcd245af6f421d5d48a15159d0a97c6fcbf251bcc721e928eae3008a031","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-27"],"prior_runs_affected":2,"prior_last_seen_at":"2026-07-28T16:11:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a3c8e0\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-4z5qknlhfz6j\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00c871f1-c55a-462f-87ca-c17ffa7cfa45\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00c871f1-c55a-462f-87ca-c17ffa7cfa45\",\n      \"name\": \"00c871f1-c55a-462f-87ca-c17ffa7cfa45\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:29.653354515Z\",\n      \"endTime\": \"2026-08-25T14:42:33.765546046Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a3c8e0\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-4z5qknlhfz6j\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00c871f1-c55a-462f-87ca-c17ffa7cfa45\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00c871f1-c55a-462f-87ca-c17ffa7cfa45\",\n      \"name\": \"00c871f1-c55a-462f-87ca-c17ffa7cfa45\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:29.653354515Z\",\n      \"endTime\": \"2026-08-25T14:42:33.765546046Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011468e0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-w7786nk5nznp\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9975191b-6ab2-42a8-85c6-83aed94ceaf0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9975191b-6ab2-42a8-85c6-83aed94ceaf0\",\n      \"name\": \"9975191b-6ab2-42a8-85c6-83aed94ceaf0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:19.963379665Z\",\n      \"endTime\": \"2026-08-25T14:42:23.660042904Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011468e0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-w7786nk5nznp\" to finish creating: GET https://rp.j9549056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9975191b-6ab2-42a8-85c6-83aed94ceaf0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9975191b-6ab2-42a8-85c6-83aed94ceaf0\",\n      \"name\": \"9975191b-6ab2-42a8-85c6-83aed94ceaf0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:23:19.963379665Z\",\n      \"endTime\": \"2026-08-25T14:42:23.660042904Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"ebe5c6af0803fc1a7dab9b5770745552c68456e64c75e2851d5fb42f7169307d","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240745749549056","occurred_at":"2026-08-25T13:20:40Z","signature_id":"eed43f59ecea085b1ffeb037fb6bd3536093190e7881447ceae828484fef5878","pr_number":6641,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b393a3822cea98a3f0355403abde765ef631da69f047c1828358b6194172cb6b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-sjzzj9kjw9kf/cluster-candidate-5-0-rjlgtk should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3a660\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-rjlgtk: failed waiting for cluster=\"cluster-candidate-5-0-rjlgtk\" in resourcegroup=\"rg-candidate-5-0-sjzzj9kjw9kf\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dd830d6-347e-4473-bb10-124104e89a73\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dd830d6-347e-4473-bb10-124104e89a73\",\n      \"name\": \"8dd830d6-347e-4473-bb10-124104e89a73\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:56.05360308Z\",\n      \"endTime\": \"2026-08-25T11:23:57.532994811Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-sjzzj9kjw9kf/cluster-candidate-5-0-rjlgtk should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3a660\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-rjlgtk: failed waiting for cluster=\"cluster-candidate-5-0-rjlgtk\" in resourcegroup=\"rg-candidate-5-0-sjzzj9kjw9kf\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dd830d6-347e-4473-bb10-124104e89a73\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dd830d6-347e-4473-bb10-124104e89a73\",\n      \"name\": \"8dd830d6-347e-4473-bb10-124104e89a73\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:56.05360308Z\",\n      \"endTime\": \"2026-08-25T11:23:57.532994811Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-wndrtf\" with version 4.20.34 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00132c740\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-wndrtf: failed waiting for cluster=\"cluster-zstream-4-20-wndrtf\" in resourcegroup=\"rg-zstream-upgrade-4-20-btw2nf2rz2z4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\",\n      \"name\": \"a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:06.275646571Z\",\n      \"endTime\": \"2026-08-25T11:24:07.685798394Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-wndrtf\" with version 4.20.34 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00132c740\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-wndrtf: failed waiting for cluster=\"cluster-zstream-4-20-wndrtf\" in resourcegroup=\"rg-zstream-upgrade-4-20-btw2nf2rz2z4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\",\n      \"name\": \"a719e0dc-6b35-4541-b9c0-4edfc6bb07b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:06.275646571Z\",\n      \"endTime\": \"2026-08-25T11:24:07.685798394Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"28b9c82589a4dda1452e567d40ab21019c6cee221e37e01472de1dab7d5143df","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"f935555cb57f59b6764c10d1c7fa203076af04b63661aa2b31d669f4c7cdb56f","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2ffeafac71db9cc68f648812079b8ded624a6470e7c1704cbbbe007755392f55","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-prc4m6\" with version 4.20.34 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001c9c3c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-prc4m6: failed waiting for cluster=\"cluster-zstream-4-20-prc4m6\" in resourcegroup=\"rg-zstream-upgrade-4-20-74gcx95r2n24\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\",\n      \"name\": \"7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:20:56.871797957Z\",\n      \"endTime\": \"2026-08-25T16:39:58.557618251Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-prc4m6\" with version 4.20.34 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001c9c3c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-prc4m6: failed waiting for cluster=\"cluster-zstream-4-20-prc4m6\" in resourcegroup=\"rg-zstream-upgrade-4-20-74gcx95r2n24\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\",\n      \"name\": \"7a916a4f-de9b-4e7b-9d68-7d520bc7ba74\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:20:56.871797957Z\",\n      \"endTime\": \"2026-08-25T16:39:58.557618251Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-zgjbj5 with version 4.20.35\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b1e160\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-zgjbj5: failed waiting for cluster=\"np-version-upgrade-cluster-zgjbj5\" in resourcegroup=\"rg-np-version-upgrade-zgjbj5-5lcrgtsl9x9p\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\",\n      \"name\": \"ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:20:59.293835394Z\",\n      \"endTime\": \"2026-08-25T16:40:08.618237862Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-zgjbj5 with version 4.20.35\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b1e160\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-zgjbj5: failed waiting for cluster=\"np-version-upgrade-cluster-zgjbj5\" in resourcegroup=\"rg-np-version-upgrade-zgjbj5-5lcrgtsl9x9p\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\",\n      \"name\": \"ccd91084-1a3a-4d2a-a6a7-9f3e0b14e716\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:20:59.293835394Z\",\n      \"endTime\": \"2026-08-25T16:40:08.618237862Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"1e71ee8a9468b4add845994e78143a38f98dc804aebda495c2ae55adfc2902b9","pr_number":6645,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"338f3c10fa302ec66d489a7e252c6f877b54dac7f42009f33bf533ea79c5069d","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7fa5ceb8cdc856d9852c9106d45ffc1d35ac035ca27214fa5e56a7315735c71c","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: applydesires, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[7,2,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-19T14:06:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001108438\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-gchzdq\" in resourcegroup=\"rg-np-skip-minor-gchzdq-ww2xhlmtfnwr\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf1121db-2117-4096-a637-d4dd852c6573\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf1121db-2117-4096-a637-d4dd852c6573\",\n      \"name\": \"bf1121db-2117-4096-a637-d4dd852c6573\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:31:16.470227528Z\",\n      \"endTime\": \"2026-08-19T15:55:21.296940766Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9h5qdf8gb4n07q0k99jcbmc1n5ti still exists (deletion dispatched at 2026-08-19T15:31:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 12 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001108438\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-gchzdq\" in resourcegroup=\"rg-np-skip-minor-gchzdq-ww2xhlmtfnwr\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf1121db-2117-4096-a637-d4dd852c6573\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf1121db-2117-4096-a637-d4dd852c6573\",\n      \"name\": \"bf1121db-2117-4096-a637-d4dd852c6573\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:31:16.470227528Z\",\n      \"endTime\": \"2026-08-19T15:55:21.296940766Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9h5qdf8gb4n07q0k99jcbmc1n5ti still exists (deletion dispatched at 2026-08-19T15:31:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 12 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012ab380\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-nf58gl6nvj9t\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ceafc180-72fe-411b-afff-85006650127a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ceafc180-72fe-411b-afff-85006650127a\",\n      \"name\": \"ceafc180-72fe-411b-afff-85006650127a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:09:54.54926661Z\",\n      \"endTime\": \"2026-08-19T15:34:00.454237581Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9l7r4r26t0e2p0ms6h9a78nd8l01 still exists (deletion dispatched at 2026-08-19T15:09:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 12 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012ab380\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-nf58gl6nvj9t\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ceafc180-72fe-411b-afff-85006650127a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ceafc180-72fe-411b-afff-85006650127a\",\n      \"name\": \"ceafc180-72fe-411b-afff-85006650127a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:09:54.54926661Z\",\n      \"endTime\": \"2026-08-19T15:34:00.454237581Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9l7r4r26t0e2p0ms6h9a78nd8l01 still exists (deletion dispatched at 2026-08-19T15:09:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 12 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"393e8fed94117bab564558b9b1b4cfb0c3f672fbd28d6d49c2152859b96335c3","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"59f960a2f1de30f6968c68d5db08823bb87f1a46ad72088af6549f8925fd368c","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c7cc4ecdb77e22f303e5ff2a71d414dfade083e8a3ba8e06dd7fd72186c1004d","failure_pattern":"route host should not be empty","search_query":"route host should not be empty","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T11:53:35Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS, private ingress, and private KeyVault and verify all are operational","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:209]: Timed out after 305.487s.\nsample app route should become available\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:208 with:\nroute host should not be empty\nExpected\n    \u003cstring\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:209]: Timed out after 305.487s.\nsample app route should become available\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:208 with:\nroute host should not be empty\nExpected\n    \u003cstring\u003e: ...\nnot to be empty","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:236]: Timed out after 304.974s.\nsample app route should become available\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:235 with:\nroute host should not be empty\nExpected\n    \u003cstring\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:236]: Timed out after 304.974s.\nsample app route should become available\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:235 with:\nroute host should not be empty\nExpected\n    \u003cstring\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090769277882732544","occurred_at":"2026-08-21T11:53:35Z","signature_id":"21340eb6aa41c283e6be7304c4a94150fcd6181a4804c245c5bebf54d04d0a5d","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090769277882732544","occurred_at":"2026-08-21T11:53:35Z","signature_id":"e6609696a4b8413bbfad821326c2f67aa530f40a06835ff069ea866a2c92cbf1","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-wcx6jgxkn8nc\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wcx6jgxkn8nc\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wcx6jgxkn8nc, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wcx6jgxkn8nc\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wcx6jgxkn8nc, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"also_in":["PROD"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":108,"prior_last_seen_at":"2026-08-19T14:35:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001590300\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-wcx6jgxkn8nc\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wcx6jgxkn8nc\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wcx6jgxkn8nc, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wcx6jgxkn8nc\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wcx6jgxkn8nc, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001590300\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-wcx6jgxkn8nc\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wcx6jgxkn8nc\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wcx6jgxkn8nc, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wcx6jgxkn8nc\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wcx6jgxkn8nc, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085264507539456","occurred_at":"2026-08-19T14:35:33Z","signature_id":"da5ef245b7e16db299546afb081bf622baf8f64d77b67033bc19a14385ca9f5d","pr_number":6607,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b55233e1f233919ecc579e6967f76823775cbaf35c5ee37e0a8883b46726c324","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AllocationFailed; detail message The VM allocation failed due to an internal error. Please retry later or try deploying to a different location.; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-10"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-16T12:18:51Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-08-24T19:10:07.135Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T19:10:08.078Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-08-24T19:10:08.961Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=6b7c37d7d05e023c4460016a815a9ccce8683c2a1a50686369f95e5de1b45b05 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6400512-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6b7c37d7d05e023c4460016a815a9ccce8683c2a1a50686369f95e5de1b45b05\ntime=2026-08-24T19:19:10.348Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/6b7c37d7d05e023c4460016a815a9ccce8683c2a1a50686369f95e5de1b45b05/operationStatuses/08584140082768889498\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/cluster-3vizzf2ahtjlc\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j6400512-mgmt-2/agentPools/userswft1\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n                    \\\\\\\"details\\\\\\\": [\\\\r\\\\n                      {\\\\r\\\\n                        \\\\\\\"code\\\\\\\": \\\\\\\"AllocationFailed\\\\\\\",\\\\r\\\\n                        \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft1-38184404-vmss'. Please try alternative sizes 'Standard_E16ds_v6' (zones 2, 3, 1), 'Standard_E16ds_v6' (eastus, zones 1, 2, 3), 'Standard_E16ds_v6' (eastus2, zones 2, 1, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation.\\\\\\\",\\\\r\\\\n                        \\\\\\\"details\\\\\\\": [\\\\r\\\\n                          {\\\\r\\\\n                            \\\\\\\"code\\\\\\\": \\\\\\\"AllocationFailed\\\\\\\",\\\\r\\\\n                            \\\\\\\"target\\\\\\\": \\\\\\\"3\\\\\\\",\\\\r\\\\n                            \\\\\\\"message\\\\\\\": \\\\\\\"The VM allocation failed due to an internal error. Please retry later or try deploying to a different location.\\\\\\\"\\\\r\\\\n                          }\\\\r\\\\n                        ]\\\\r\\\\n                      }\\\\r\\\\n                    ]\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091957267166400512","occurred_at":"2026-08-24T18:34:13Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"50b4072821c611f78915fffb00974b54efe31284ac524891f02603d30b54dc67","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message The requested operation cannot be performed because the database account arohcpci01-rp-j2394624 is in the process of being created. Please wait until account creation completes and retry the operation.\\\\\\\\\\\\\\ \\\\\\\\\\\\ ActivityId: \u003cuuid\u003e, Microsoft.Azure.Documents.Common/2.14.0\"\\\\\\ \\\\ }; provider Microsoft.DocumentDB","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T20:51:11Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1}],"full_error_samples":["time=2026-08-19T21:09:01.009Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-08-19T21:09:03.163Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-08-19T21:09:04.592Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=5d7d7fd1d17d5cb539e3ba4395d742a56d454dcabf04b8e8cd3c81884c4e29c2 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2394624%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F5d7d7fd1d17d5cb539e3ba4395d742a56d454dcabf04b8e8cd3c81884c4e29c2\ntime=2026-08-19T21:11:05.314Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2394624/providers/Microsoft.Resources/deployments/5d7d7fd1d17d5cb539e3ba4395d742a56d454dcabf04b8e8cd3c81884c4e29c2/operationStatuses/08584144331418179698\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2394624/providers/Microsoft.Resources/deployments/rp-cosmos-account\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2394624/providers/Microsoft.DocumentDB/databaseAccounts/arohcpci01-rp-j2394624\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"BadRequest\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"The requested operation cannot be performed because the database account arohcpci01-rp-j2394624 is in the process of being created. Please wait until account creation completes and retry the operation.\\\\\\\\\\\\\\\\r\\\\\\\\\\\\\\\\nActivityId: 02191ce0-56b4-4967-b294-a61f839cd58d, Microsoft.Azure.Documents.Common/2.14.0\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090179795802394624","occurred_at":"2026-08-19T20:51:11Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":6492,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cb9a7ba2d967015ac741b1ba2401e95d8184541adbccfc98565936e10e7aa68c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ConcurrentFederatedIdentityCredentialsWritesForSingleManagedIdentity; detail message Too many Federated Identity Credentials are written concurrently for the managed identity '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/microsoft.managedidentity/userassignedidentities/fleet'. Concurrent Federated Identity Credentials writes under the same managed identity are not supported.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T17:25:51Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1}],"full_error_samples":["time=2026-08-21T17:53:50.354Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-21T17:53:52.743Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-21T17:53:54.610Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=3dbafda6428a192bbb810e342f38cdf4f95c2f9f6f0490ebf2e0d35b2ec09729 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9453824-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F3dbafda6428a192bbb810e342f38cdf4f95c2f9f6f0490ebf2e0d35b2ec09729\ntime=2026-08-21T18:05:27.260Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9453824-svc/providers/Microsoft.Resources/deployments/3dbafda6428a192bbb810e342f38cdf4f95c2f9f6f0490ebf2e0d35b2ec09729/operationStatuses/08584142720518233456\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9453824-svc/providers/Microsoft.Resources/deployments/cluster-bptwqnbhgueys\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ConcurrentFederatedIdentityCredentialsWritesForSingleManagedIdentity\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Too many Federated Identity Credentials are written concurrently for the managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9453824-svc/providers/microsoft.managedidentity/userassignedidentities/fleet'. Concurrent Federated Identity Credentials writes under the same managed identity are not supported.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6650/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090852896869453824","occurred_at":"2026-08-21T17:25:51Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6650,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cb3f1af3ae5c6870302b8493e6567a2cb5b04bdc76461d8531d6cdaf4f74aada","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ConflictError; detail message A conflict occurred that prevented the operation from completing. The operation failed because the Key Vault '' changed from the point the operation began. This can happen if parallel operations are being performed on the Key Vault. To prevent this error, serialize the operations so that only one operation is performed on the Key Vault at a time. Follow this link for more information: \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-03"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-07T00:57:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e56140\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-np-update-nodes-hcp-cluster-chtjsv\" in resourcegroup=\"nodepool-update-nodes-sz99wmd6668j\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/nodepool-update-nodes-sz99wmd6668j/providers/Microsoft.Resources/deployments/customer-infra-np-update-nodes-hcp-cluster-chtjsv/operationStatuses/08584139228279467438\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ConflictError\\\",\\r\\n    \\\"message\\\": \\\"A conflict occurred that prevented the operation from completing. The operation failed because the Key Vault '' changed from the point the operation began. This can happen if parallel operations are being performed on the Key Vault. To prevent this error, serialize the operations so that only one operation is performed on the Key Vault at a time. Follow this link for more information: https://go.microsoft.com/fwlink/?linkid=2147741\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:73]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e56140\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-np-update-nodes-hcp-cluster-chtjsv\" in resourcegroup=\"nodepool-update-nodes-sz99wmd6668j\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/nodepool-update-nodes-sz99wmd6668j/providers/Microsoft.Resources/deployments/customer-infra-np-update-nodes-hcp-cluster-chtjsv/operationStatuses/08584139228279467438\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ConflictError\\\",\\r\\n    \\\"message\\\": \\\"A conflict occurred that prevented the operation from completing. The operation failed because the Key Vault '' changed from the point the operation began. This can happen if parallel operations are being performed on the Key Vault. To prevent this error, serialize the operations so that only one operation is performed on the Key Vault at a time. Follow this link for more information: https://go.microsoft.com/fwlink/?linkid=2147741\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314114058620928","occurred_at":"2026-08-25T18:12:12Z","signature_id":"6fabb5bf9236add92797cdcc42cf1ddf33539333a8c7650c981d9fd3ebea687d","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8097f2fb40f9dda7135748f56dd66bc27b12f2b97919326dd7b3b4c74d914621","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceConflict; detail message ETag mismatch on \u003cuuid\u003e; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T05:59:26Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1}],"full_error_samples":["time=2026-08-21T06:22:09.432Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-08-21T06:22:11.097Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-08-21T06:22:13.018Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=e422eaa0da3d95490d2441b44a33daba3f65b59ac89f70d027fcb6e1074a2f11 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2081536%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fe422eaa0da3d95490d2441b44a33daba3f65b59ac89f70d027fcb6e1074a2f11\ntime=2026-08-21T06:28:14.734Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2081536/providers/Microsoft.Resources/deployments/e422eaa0da3d95490d2441b44a33daba3f65b59ac89f70d027fcb6e1074a2f11/operationStatuses/08584143135538485032\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2081536/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2081536/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j2081536\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"ResourceConflict\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"ETag mismatch on 28111326-98a3-4ad2-87d8-5c0102bdf0cd\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090680154262081536","occurred_at":"2026-08-21T05:59:26Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":6640,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bb424b290c173083428a34ac21b9472b0a9be4d1c592b771df2c71e46b269465","failure_pattern":"ERROR CODE: DeploymentFailed; detail code StorageAccountOperationInProgress; detail message An operation is currently performing on this storage account that requires exclusive access.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-08-24T16:09:49.176Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 description=\"Step infra\\n  Kind: ARM\\n  Template: templates/mgmt-infra.bicep\\n  Parameters: configurations/mgmt-infra.tmpl.bicepparam\"\ntime=2026-08-24T16:09:49.840Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2\ntime=2026-08-24T16:09:50.553Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 deployment=c783e3e3d697ee4d22dffbab4271479e105f154955a91c3a47e9819df57ac290 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1968512-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc783e3e3d697ee4d22dffbab4271479e105f154955a91c3a47e9819df57ac290\ntime=2026-08-24T16:11:20.899Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/c783e3e3d697ee4d22dffbab4271479e105f154955a91c3a47e9819df57ac290/operationStatuses/08584140190952500816\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/hcp-backups-storage\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/hcpBackupsStorageAccount\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/hcpBackupsStorageAccount\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"StorageAccountOperationInProgress\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"An operation is currently performing on this storage account that requires exclusive access.\\\\\\\"\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6659/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091914307531968512","occurred_at":"2026-08-24T15:43:31Z","signature_id":"d517e51fc5b09c819903bacd247416bcbbe167939922787f3a2438a125b3fdc2","pr_number":6659,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ca6140fe6832e567dbdbd62e69723ab444b3c85ac16836f37268901fc2e842b0","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Timeout; detail message Operation is being Canceled due to timeout.; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1}],"full_error_samples":["time=2026-08-24T07:13:06.981Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-08-24T07:13:09.018Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-08-24T07:13:10.536Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9665ebaf931e927bcf7f137293ddd5a9262983f1f2075a0dff43a22749c6dc59 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1860736%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9665ebaf931e927bcf7f137293ddd5a9262983f1f2075a0dff43a22749c6dc59\ntime=2026-08-24T07:29:44.382Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1860736/providers/Microsoft.Resources/deployments/9665ebaf931e927bcf7f137293ddd5a9262983f1f2075a0dff43a22749c6dc59/operationStatuses/08584140512958847265\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1860736/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1860736/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1860736\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Timeout\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Operation is being Canceled due to timeout.\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091780518591860736","occurred_at":"2026-08-24T06:51:53Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":6278,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"36e7932077fe0de3697c04fb0ba302a7298b56f122f6c49db035b3d764e4980b","failure_pattern":"ERROR CODE: DeploymentStackDeploymentFailed; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentStackDeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nodepools (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nodepools (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-08-24T08:15:25.803Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nodepools stamp=2 description=\"Step nodepools\\n  Kind: ARMStack\\n  Template: templates/aks-nodepools.bicep\\n  Parameters: configurations/mgmt-cluster-nodepools.tmpl.bicepparam\"\ntime=2026-08-24T08:18:16.150Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nodepools stamp=2 err=\"stamp 2: failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/67d8fd4b-85de-4d57-a829-7ce0bf268c5d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/67d8fd4b-85de-4d57-a829-7ce0bf268c5d\\\",\\n  \\\"name\\\": \\\"67d8fd4b-85de-4d57-a829-7ce0bf268c5d\\\",\\n  \\\"status\\\": \\\"failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n    \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: '676c3fe4-b0a1-44f5-8602-c3c05012b379'.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9428224-mgmt-2/providers/Microsoft.Resources/deployments/l4mv3scto7z6edigthkjdr62jbdrgoxoif33nvakxlwvg-260824084ipd0\\\",\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n        \\\"details\\\": [\\n          {\\n            \\\"code\\\": \\\"OverconstrainedZonalAllocationRequest\\\",\\n            \\\"message\\\": \\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9428224-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-41975975-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9428224-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-41975975-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n  - Availability Zone\\\\n  - Differencing (Ephemeral) Disks\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n  - VM Size\\\\n\\\"\\n          }\\n        ]\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091793135989428224","occurred_at":"2026-08-24T07:42:01Z","signature_id":"497ab79e6fe621da7f982b21f5dd28a45dc2f63297d3ed53abd84200879145f5","pr_number":6447,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"72de96d5fccd2e929d86c234bb6122fb3e96395118e1ddf6c2c21fc7fb224f37","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101a160\u003e: \n    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-qmzd7vdsthg7\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1cf37f1-e335-4a31-8459-8927106e9275\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1cf37f1-e335-4a31-8459-8927106e9275\",\n      \"name\": \"d1cf37f1-e335-4a31-8459-8927106e9275\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:20.481967932Z\",\n      \"endTime\": \"2026-08-25T19:39:22.436659526Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101a160\u003e: \n    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-qmzd7vdsthg7\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1cf37f1-e335-4a31-8459-8927106e9275\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1cf37f1-e335-4a31-8459-8927106e9275\",\n      \"name\": \"d1cf37f1-e335-4a31-8459-8927106e9275\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:20.481967932Z\",\n      \"endTime\": \"2026-08-25T19:39:22.436659526Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"d496820b37cab7fa1ea7b0c47e837d533327b2947405aa862fe6cf0becda824e","pr_number":6683,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2bdf0d0e26486c8baecfb566f0a7ae932a4a2cee72db742fb786cc445dbea21e","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, catalog-operator, certified-operators-catalog, cluster-autoscaler, cluster-image-registry-operator, cluster-network-operator, cluster-node-tuning-operator, cluster-policy-controller, cluster-storage-operator, cluster-version-operator, community-operators-catalog, control-plane-pki-operator, csi-snapshot-controller-operator, dns-operator, hosted-cluster-config-operator, ignition-server, ignition-server-proxy, ingress-operator, konnectivity-agent, kube-controller-manager, kube-scheduler, machine-approver, olm-collect-profiles, olm-operator, openshift-apiserver, openshift-controller-manager, openshift-route-controller-manager, packageserver, redhat-marketplace-catalog, redhat-operators-catalog; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010fa820\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-gp4rdpwm7zhd\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ee129a2-c81c-4d02-bea3-92db5bdcc541\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ee129a2-c81c-4d02-bea3-92db5bdcc541\",\n      \"name\": \"5ee129a2-c81c-4d02-bea3-92db5bdcc541\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:55.463575249Z\",\n      \"endTime\": \"2026-08-25T11:23:57.522671815Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, olm-collect-profiles, cluster-policy-controller, cluster-node-tuning-operator, certified-operators-catalog, cluster-storage-operator, dns-operator, cluster-image-registry-operator, packageserver, hosted-cluster-config-operator, cluster-network-operator, ignition-server-proxy, catalog-operator, openshift-apiserver, ignition-server, olm-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, machine-approver, kube-controller-manager, openshift-controller-manager, csi-snapshot-controller-operator, control-plane-pki-operator, kube-scheduler, cluster-version-operator, cluster-autoscaler, openshift-route-controller-manager, azure-cloud-controller-manager, ingress-operator\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010fa820\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-gp4rdpwm7zhd\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ee129a2-c81c-4d02-bea3-92db5bdcc541\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ee129a2-c81c-4d02-bea3-92db5bdcc541\",\n      \"name\": \"5ee129a2-c81c-4d02-bea3-92db5bdcc541\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:55.463575249Z\",\n      \"endTime\": \"2026-08-25T11:23:57.522671815Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent, olm-collect-profiles, cluster-policy-controller, cluster-node-tuning-operator, certified-operators-catalog, cluster-storage-operator, dns-operator, cluster-image-registry-operator, packageserver, hosted-cluster-config-operator, cluster-network-operator, ignition-server-proxy, catalog-operator, openshift-apiserver, ignition-server, olm-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, machine-approver, kube-controller-manager, openshift-controller-manager, csi-snapshot-controller-operator, control-plane-pki-operator, kube-scheduler, cluster-version-operator, cluster-autoscaler, openshift-route-controller-manager, azure-cloud-controller-manager, ingress-operator\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"66ee2372a7d35bafc2a7139acace8a079bd17442b7e81b824c6f13e5ea4be15f","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b3d328e9a97f3f976227e1f892def10b7f6c53bda848b798f9760113f189d2a4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-network-operator, ingress-operator; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001a2ee20\u003e: \n    failed to create HCP cluster np-dg-minor-8wczf9: failed waiting for cluster=\"np-dg-minor-8wczf9\" in resourcegroup=\"rg-np-dg-minor-8wczf9-z98rv7l55r2s\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35e222d0-bdde-4e12-a2e6-3658ebb699b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35e222d0-bdde-4e12-a2e6-3658ebb699b6\",\n      \"name\": \"35e222d0-bdde-4e12-a2e6-3658ebb699b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:58.632569496Z\",\n      \"endTime\": \"2026-08-25T11:24:07.68601483Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator, cluster-image-registry-operator\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001a2ee20\u003e: \n    failed to create HCP cluster np-dg-minor-8wczf9: failed waiting for cluster=\"np-dg-minor-8wczf9\" in resourcegroup=\"rg-np-dg-minor-8wczf9-z98rv7l55r2s\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35e222d0-bdde-4e12-a2e6-3658ebb699b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35e222d0-bdde-4e12-a2e6-3658ebb699b6\",\n      \"name\": \"35e222d0-bdde-4e12-a2e6-3658ebb699b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:58.632569496Z\",\n      \"endTime\": \"2026-08-25T11:24:07.68601483Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator, cluster-image-registry-operator\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"3286e3629eaf6809d169cb339959b7df55417659272206300cd3912a2d082c76","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"92ae44f2e86af5544dec422ded7c331d2f560f22fa4954538768a565c71a9dbd","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e84820\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-b9lzcpbpn5wp\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2bbcb592-de1c-4ea4-8806-5849cdf0735a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2bbcb592-de1c-4ea4-8806-5849cdf0735a\",\n      \"name\": \"2bbcb592-de1c-4ea4-8806-5849cdf0735a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:06.392903067Z\",\n      \"endTime\": \"2026-08-25T11:24:07.668839785Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e84820\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-b9lzcpbpn5wp\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2bbcb592-de1c-4ea4-8806-5849cdf0735a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2bbcb592-de1c-4ea4-8806-5849cdf0735a\",\n      \"name\": \"2bbcb592-de1c-4ea4-8806-5849cdf0735a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:06.392903067Z\",\n      \"endTime\": \"2026-08-25T11:24:07.668839785Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"a34611ae2bd1afe363e9b85e5d346d6e037222fe5510e7c418c4c0d0a2de08f2","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8a80f07fe2ebdec087cc11f27aeb724cf8c6cf8aa029e8f467e1f89fd185112d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-22T00:25:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-kkp4qflqb9tl/cluster-candidate-5-0-nzjftt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f14860\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-nzjftt: failed waiting for cluster=\"cluster-candidate-5-0-nzjftt\" in resourcegroup=\"rg-candidate-5-0-kkp4qflqb9tl\" to finish creating: GET https://rp.j3592192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/879d27b2-7910-474a-ade0-ae3e7eab5f6e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/879d27b2-7910-474a-ade0-ae3e7eab5f6e\",\n      \"name\": \"879d27b2-7910-474a-ade0-ae3e7eab5f6e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:19:04.482166059Z\",\n      \"endTime\": \"2026-08-22T01:38:11.22344033Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-kkp4qflqb9tl/cluster-candidate-5-0-nzjftt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f14860\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-nzjftt: failed waiting for cluster=\"cluster-candidate-5-0-nzjftt\" in resourcegroup=\"rg-candidate-5-0-kkp4qflqb9tl\" to finish creating: GET https://rp.j3592192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/879d27b2-7910-474a-ade0-ae3e7eab5f6e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/879d27b2-7910-474a-ade0-ae3e7eab5f6e\",\n      \"name\": \"879d27b2-7910-474a-ade0-ae3e7eab5f6e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:19:04.482166059Z\",\n      \"endTime\": \"2026-08-22T01:38:11.22344033Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090958470403592192","occurred_at":"2026-08-22T00:25:22Z","signature_id":"201d37275ac85dc0beaa94be1f84bc640b3be944f1830b8c0efc34b279522a24","pr_number":6554,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e339823035a3b42061d546c2e6cbdec3cd57b33bd40fddf5adef300f46092d74","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, openshift-controller-manager; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001081c80\u003e: \n    GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7baee5b-db11-4464-aa8e-6e2d27501249\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7baee5b-db11-4464-aa8e-6e2d27501249\",\n      \"name\": \"f7baee5b-db11-4464-aa8e-6e2d27501249\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:06:19.502868721Z\",\n      \"endTime\": \"2026-08-25T11:25:27.633996529Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-node-tuning-operator\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001081c80\u003e: \n    GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7baee5b-db11-4464-aa8e-6e2d27501249\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f7baee5b-db11-4464-aa8e-6e2d27501249\",\n      \"name\": \"f7baee5b-db11-4464-aa8e-6e2d27501249\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:06:19.502868721Z\",\n      \"endTime\": \"2026-08-25T11:25:27.633996529Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-node-tuning-operator\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"62ef36a640529db2357933eec4cfc1865780247b38f5766b5fa5e83338b90158","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7be6b09c6bbb28adba99c6de7dfc1604dfce000bc6664dedbae8e8b9b0253fa6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006eefa0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-blffsk8cqz4z\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cb49989-b0d0-4959-b0b9-b9f58628cdd6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cb49989-b0d0-4959-b0b9-b9f58628cdd6\",\n      \"name\": \"7cb49989-b0d0-4959-b0b9-b9f58628cdd6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:07.01277565Z\",\n      \"endTime\": \"2026-08-25T11:24:07.575415707Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006eefa0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-blffsk8cqz4z\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cb49989-b0d0-4959-b0b9-b9f58628cdd6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cb49989-b0d0-4959-b0b9-b9f58628cdd6\",\n      \"name\": \"7cb49989-b0d0-4959-b0b9-b9f58628cdd6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:07.01277565Z\",\n      \"endTime\": \"2026-08-25T11:24:07.575415707Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"20fc32ee45196a9ed01e8c936d2dde739ba00031d099ddd39be043bfecacc1ed","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ab3cc8ce178fa5fcf5b2293012368a7b35c8796690239f588635f09f67db92fe","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007fc3a0\u003e: \n    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-7qr482mpm7kt\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7872695-995d-44a9-aff2-649d7b0c3f98\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7872695-995d-44a9-aff2-649d7b0c3f98\",\n      \"name\": \"b7872695-995d-44a9-aff2-649d7b0c3f98\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:39.045350362Z\",\n      \"endTime\": \"2026-08-25T11:23:42.92215819Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007fc3a0\u003e: \n    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-7qr482mpm7kt\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7872695-995d-44a9-aff2-649d7b0c3f98\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b7872695-995d-44a9-aff2-649d7b0c3f98\",\n      \"name\": \"b7872695-995d-44a9-aff2-649d7b0c3f98\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:39.045350362Z\",\n      \"endTime\": \"2026-08-25T11:23:42.92215819Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"f6f2233f1410b658d6612e4e90347e1d367b288e857c9f69f6fd3585012f8aab","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e367f2a3465740bc9b9da68a2f81e3714490aba74464e39ae8e5c3a1ae5bab94","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, kube-storage-version-migrator, machine-approver, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-p7lpzhts95vd/cluster-candidate-4-22-hlfnj9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005ad040\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-hlfnj9: failed waiting for cluster=\"cluster-candidate-4-22-hlfnj9\" in resourcegroup=\"rg-candidate-4-22-p7lpzhts95vd\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18015e61-c049-41a2-809f-ad31666a4af5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18015e61-c049-41a2-809f-ad31666a4af5\",\n      \"name\": \"18015e61-c049-41a2-809f-ad31666a4af5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:40.338382698Z\",\n      \"endTime\": \"2026-08-25T11:20:48.26963606Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-image-registry-operator, ingress-operator, kube-storage-version-migrator, catalog-operator, packageserver, certified-operators-catalog, csi-snapshot-controller-operator, community-operators-catalog, cluster-version-operator, azure-cloud-controller-manager, cluster-storage-operator, redhat-operators-catalog, cluster-policy-controller, cluster-network-operator, machine-approver, dns-operator, olm-operator, openshift-apiserver, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-p7lpzhts95vd/cluster-candidate-4-22-hlfnj9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005ad040\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-hlfnj9: failed waiting for cluster=\"cluster-candidate-4-22-hlfnj9\" in resourcegroup=\"rg-candidate-4-22-p7lpzhts95vd\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18015e61-c049-41a2-809f-ad31666a4af5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18015e61-c049-41a2-809f-ad31666a4af5\",\n      \"name\": \"18015e61-c049-41a2-809f-ad31666a4af5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:40.338382698Z\",\n      \"endTime\": \"2026-08-25T11:20:48.26963606Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-image-registry-operator, ingress-operator, kube-storage-version-migrator, catalog-operator, packageserver, certified-operators-catalog, csi-snapshot-controller-operator, community-operators-catalog, cluster-version-operator, azure-cloud-controller-manager, cluster-storage-operator, redhat-operators-catalog, cluster-policy-controller, cluster-network-operator, machine-approver, dns-operator, olm-operator, openshift-apiserver, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"312f9db8ee161a9fc497742dfc7136dbb8ed546241173b0109c0ff6c461ac5e3","pr_number":6670,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6e365d99bf1fe7b4ecb0645e26216599f71568b4b9def38c7d2aa9626ca4060b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, kube-storage-version-migrator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-6qqtw5xzzhgg/cluster-candidate-5-0-skdztd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ae4220\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-skdztd: failed waiting for cluster=\"cluster-candidate-5-0-skdztd\" in resourcegroup=\"rg-candidate-5-0-6qqtw5xzzhgg\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\",\n      \"name\": \"bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:58.368910284Z\",\n      \"endTime\": \"2026-08-25T11:22:03.073725302Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, cluster-storage-operator, cluster-version-operator, cluster-policy-controller, csi-snapshot-controller-operator, kube-storage-version-migrator, olm-operator, redhat-marketplace-catalog, cluster-node-tuning-operator, openshift-apiserver, hosted-cluster-config-operator, cluster-network-operator, community-operators-catalog, ingress-operator, catalog-operator, openshift-route-controller-manager, dns-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-6qqtw5xzzhgg/cluster-candidate-5-0-skdztd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ae4220\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-skdztd: failed waiting for cluster=\"cluster-candidate-5-0-skdztd\" in resourcegroup=\"rg-candidate-5-0-6qqtw5xzzhgg\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\",\n      \"name\": \"bd78ff65-5a1c-4595-9723-7a61e4e6ff8e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:58.368910284Z\",\n      \"endTime\": \"2026-08-25T11:22:03.073725302Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, cluster-storage-operator, cluster-version-operator, cluster-policy-controller, csi-snapshot-controller-operator, kube-storage-version-migrator, olm-operator, redhat-marketplace-catalog, cluster-node-tuning-operator, openshift-apiserver, hosted-cluster-config-operator, cluster-network-operator, community-operators-catalog, ingress-operator, catalog-operator, openshift-route-controller-manager, dns-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, cluster-image-registry-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-storage-version-migrator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"822b64d629b6a8eaf1077aec1bbfe1cde30b863a89d30905f961017f5f5f8da1","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"04ab38fb4caffee001686eb32195645f728f8555d294ffa19de0a9f505223827","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--mdtqq6tfggzt/cluster-hs-pre-5md8sn should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00138c380\u003e: \n    failed to create HCP cluster cluster-hs-pre-5md8sn: failed waiting for cluster=\"cluster-hs-pre-5md8sn\" in resourcegroup=\"rg-hs-pre--mdtqq6tfggzt\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/faa7138f-d176-4225-9189-5771c1682daa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/faa7138f-d176-4225-9189-5771c1682daa\",\n      \"name\": \"faa7138f-d176-4225-9189-5771c1682daa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:52.650546675Z\",\n      \"endTime\": \"2026-08-25T11:20:57.345950087Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, ingress-operator, redhat-marketplace-catalog, cluster-policy-controller, azure-cloud-controller-manager, cluster-image-registry-operator, certified-operators-catalog, packageserver, cluster-version-operator, cluster-storage-operator, redhat-operators-catalog, csi-snapshot-controller-operator, openshift-controller-manager, cluster-node-tuning-operator, openshift-apiserver, catalog-operator, community-operators-catalog, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-network-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--mdtqq6tfggzt/cluster-hs-pre-5md8sn should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00138c380\u003e: \n    failed to create HCP cluster cluster-hs-pre-5md8sn: failed waiting for cluster=\"cluster-hs-pre-5md8sn\" in resourcegroup=\"rg-hs-pre--mdtqq6tfggzt\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/faa7138f-d176-4225-9189-5771c1682daa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/faa7138f-d176-4225-9189-5771c1682daa\",\n      \"name\": \"faa7138f-d176-4225-9189-5771c1682daa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:52.650546675Z\",\n      \"endTime\": \"2026-08-25T11:20:57.345950087Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, ingress-operator, redhat-marketplace-catalog, cluster-policy-controller, azure-cloud-controller-manager, cluster-image-registry-operator, certified-operators-catalog, packageserver, cluster-version-operator, cluster-storage-operator, redhat-operators-catalog, csi-snapshot-controller-operator, openshift-controller-manager, cluster-node-tuning-operator, openshift-apiserver, catalog-operator, community-operators-catalog, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-network-operator, olm-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"6cfd6ef557fef8c9d185efe4042060278e1497ede40310252790075002d9aac9","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d2829443a4ebdae768869c26f5c53cd707f73b967a4f58e299f0ab62d07b2457","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, cluster-image-registry-operator, cluster-network-operator, community-operators-catalog, ingress-operator, olm-operator, packageserver, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-22T10:03:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000da51e0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-d746nz7xlk2s\" to finish creating: GET https://rp.j3428992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5fe3108e-0452-4336-9476-4c9bc112740c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5fe3108e-0452-4336-9476-4c9bc112740c\",\n      \"name\": \"5fe3108e-0452-4336-9476-4c9bc112740c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T10:45:48.497379279Z\",\n      \"endTime\": \"2026-08-22T11:04:48.540876115Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, packageserver, openshift-controller-manager, cluster-policy-controller, dns-operator, certified-operators-catalog, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-route-controller-manager, ingress-operator, cluster-image-registry-operator, catalog-operator, olm-operator, cluster-version-operator, hosted-cluster-config-operator, community-operators-catalog, cluster-network-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000da51e0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-d746nz7xlk2s\" to finish creating: GET https://rp.j3428992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5fe3108e-0452-4336-9476-4c9bc112740c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5fe3108e-0452-4336-9476-4c9bc112740c\",\n      \"name\": \"5fe3108e-0452-4336-9476-4c9bc112740c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T10:45:48.497379279Z\",\n      \"endTime\": \"2026-08-22T11:04:48.540876115Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, packageserver, openshift-controller-manager, cluster-policy-controller, dns-operator, certified-operators-catalog, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-route-controller-manager, ingress-operator, cluster-image-registry-operator, catalog-operator, olm-operator, cluster-version-operator, hosted-cluster-config-operator, community-operators-catalog, cluster-network-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091103857353428992","occurred_at":"2026-08-22T10:03:05Z","signature_id":"dc24633d9e5d820847c90cb66faabc819f731ccb727551969b9a5ee3eb7f3c9b","pr_number":6651,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"db4713748d0354976aeef8212e3a9f4a93e3ecf500342fbbcb1fbd6a21579f7d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, cluster-network-operator, csi-snapshot-controller-operator, dns-operator, hosted-cluster-config-operator, ingress-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea1740\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-jm4vsb7vn4sd\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afbb1cd8-1c39-42be-9ff7-787ffc89e069\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afbb1cd8-1c39-42be-9ff7-787ffc89e069\",\n      \"name\": \"afbb1cd8-1c39-42be-9ff7-787ffc89e069\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:22.824480944Z\",\n      \"endTime\": \"2026-08-25T11:22:27.407570836Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, csi-snapshot-controller-operator, cluster-version-operator, olm-operator, dns-operator, catalog-operator, cluster-storage-operator, ingress-operator, hosted-cluster-config-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea1740\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-jm4vsb7vn4sd\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afbb1cd8-1c39-42be-9ff7-787ffc89e069\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/afbb1cd8-1c39-42be-9ff7-787ffc89e069\",\n      \"name\": \"afbb1cd8-1c39-42be-9ff7-787ffc89e069\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:22.824480944Z\",\n      \"endTime\": \"2026-08-25T11:22:27.407570836Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, csi-snapshot-controller-operator, cluster-version-operator, olm-operator, dns-operator, catalog-operator, cluster-storage-operator, ingress-operator, hosted-cluster-config-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"a0b46f2049c857f9b50193bfa943a46b002ac4931192ad3d996324be47f034d8","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a7bb05c760ce6e02186100a06d53cbce3c5ec829fc4f0c3e2e6bbbb1e0d7f73e","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, cluster-version-operator, hosted-cluster-config-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001095660\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-b4qlj57lcxr8\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\",\n      \"name\": \"c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:28.224401746Z\",\n      \"endTime\": \"2026-08-25T11:24:37.557730725Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, cluster-network-operator, community-operators-catalog, certified-operators-catalog, cluster-policy-controller, olm-operator, packageserver, redhat-operators-catalog, dns-operator, ingress-operator, cluster-version-operator, openshift-controller-manager, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001095660\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-b4qlj57lcxr8\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\",\n      \"name\": \"c3ad12b2-96cb-4af1-a3eb-a6c632d7eb05\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:28.224401746Z\",\n      \"endTime\": \"2026-08-25T11:24:37.557730725Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator, cluster-network-operator, community-operators-catalog, certified-operators-catalog, cluster-policy-controller, olm-operator, packageserver, redhat-operators-catalog, dns-operator, ingress-operator, cluster-version-operator, openshift-controller-manager, csi-snapshot-controller-operator, cluster-image-registry-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"e3135c78719260f8b136dccd2d27e839bcb88513840553199c7a5a72c9c78af4","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c97c7e10e9984956816f2978cee06c0347355fb43e2e18dbc518e10a50fd697c","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, machine-approver, olm-operator, openshift-apiserver, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--2zj5qrnqfwdn/cluster-hs-pre-4n4p2k should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007e89c0\u003e: \n    failed to create HCP cluster cluster-hs-pre-4n4p2k: failed waiting for cluster=\"cluster-hs-pre-4n4p2k\" in resourcegroup=\"rg-hs-pre--2zj5qrnqfwdn\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7939f0c1-73a2-48bf-80a1-c5a81f59060f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7939f0c1-73a2-48bf-80a1-c5a81f59060f\",\n      \"name\": \"7939f0c1-73a2-48bf-80a1-c5a81f59060f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:28.656859519Z\",\n      \"endTime\": \"2026-08-25T11:20:37.93508057Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-policy-controller, cluster-storage-operator, cluster-node-tuning-operator, dns-operator, cluster-image-registry-operator, packageserver, certified-operators-catalog, hosted-cluster-config-operator, olm-operator, openshift-route-controller-manager, openshift-controller-manager, catalog-operator, cluster-network-operator, machine-approver, csi-snapshot-controller-operator, community-operators-catalog, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--2zj5qrnqfwdn/cluster-hs-pre-4n4p2k should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007e89c0\u003e: \n    failed to create HCP cluster cluster-hs-pre-4n4p2k: failed waiting for cluster=\"cluster-hs-pre-4n4p2k\" in resourcegroup=\"rg-hs-pre--2zj5qrnqfwdn\" to finish creating: GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7939f0c1-73a2-48bf-80a1-c5a81f59060f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7939f0c1-73a2-48bf-80a1-c5a81f59060f\",\n      \"name\": \"7939f0c1-73a2-48bf-80a1-c5a81f59060f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:28.656859519Z\",\n      \"endTime\": \"2026-08-25T11:20:37.93508057Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-policy-controller, cluster-storage-operator, cluster-node-tuning-operator, dns-operator, cluster-image-registry-operator, packageserver, certified-operators-catalog, hosted-cluster-config-operator, olm-operator, openshift-route-controller-manager, openshift-controller-manager, catalog-operator, cluster-network-operator, machine-approver, csi-snapshot-controller-operator, community-operators-catalog, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"62b66f3cda1a1b03c5a9f22a0c0fb4bfb068df7258a21d08a9456f0d47c984ee","pr_number":6670,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e258b478ac9644b09eb20a3a01941c7945db5504ffd2b81616ca149fbb4626ea","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, machine-approver, olm-operator, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0019fc320\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-pvw94n8jrfrs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17ba9133-3d45-4498-9ddf-cf4f2ab615f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17ba9133-3d45-4498-9ddf-cf4f2ab615f6\",\n      \"name\": \"17ba9133-3d45-4498-9ddf-cf4f2ab615f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:18:50.270808871Z\",\n      \"endTime\": \"2026-08-25T19:37:52.396630467Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, csi-snapshot-controller-operator, packageserver, machine-approver, cluster-policy-controller, cluster-image-registry-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-network-operator, ingress-operator, catalog-operator, olm-operator, community-operators-catalog, dns-operator, cluster-node-tuning-operator, redhat-operators-catalog, redhat-marketplace-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-version-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0019fc320\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-pvw94n8jrfrs\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17ba9133-3d45-4498-9ddf-cf4f2ab615f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/17ba9133-3d45-4498-9ddf-cf4f2ab615f6\",\n      \"name\": \"17ba9133-3d45-4498-9ddf-cf4f2ab615f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:18:50.270808871Z\",\n      \"endTime\": \"2026-08-25T19:37:52.396630467Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, csi-snapshot-controller-operator, packageserver, machine-approver, cluster-policy-controller, cluster-image-registry-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-network-operator, ingress-operator, catalog-operator, olm-operator, community-operators-catalog, dns-operator, cluster-node-tuning-operator, redhat-operators-catalog, redhat-marketplace-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-version-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"3a165bfaf5767e237e81c7162c0c48f850249eb7416a5043163724820b2a6d91","pr_number":6683,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"42b2d5dca43e9e034a17b7b121f324959a41ced09de1009bebe039aef5a32bf2","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, cluster-image-registry-operator, cluster-node-tuning-operator, cluster-policy-controller, csi-snapshot-controller-operator, ignition-server-proxy, ingress-operator, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T18:39:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001004c60\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2mn5zf9jcbq8\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80ff098e-3bea-479a-9923-3c6d996be86d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80ff098e-3bea-479a-9923-3c6d996be86d\",\n      \"name\": \"80ff098e-3bea-479a-9923-3c6d996be86d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:41.976383533Z\",\n      \"endTime\": \"2026-08-19T19:47:49.788162245Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, cluster-node-tuning-operator, cluster-image-registry-operator, redhat-operators-catalog, ingress-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001004c60\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2mn5zf9jcbq8\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80ff098e-3bea-479a-9923-3c6d996be86d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80ff098e-3bea-479a-9923-3c6d996be86d\",\n      \"name\": \"80ff098e-3bea-479a-9923-3c6d996be86d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:41.976383533Z\",\n      \"endTime\": \"2026-08-19T19:47:49.788162245Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, cluster-node-tuning-operator, cluster-image-registry-operator, redhat-operators-catalog, ingress-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"97c529fbcfca706ccd9d033fcefb76e6725645f13e56e9dbe948031baa2eefc4","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"56ac1ffd8f7c84d6800647347b139a137e691a55ff2e3d1faa305a6b07101364","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-x7w9rq with version 4.20.35\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000218ae0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-x7w9rq: failed waiting for cluster=\"np-version-upgrade-cluster-x7w9rq\" in resourcegroup=\"rg-np-version-upgrade-x7w9rq-dvf8r82dxzzq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\",\n      \"name\": \"96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:57.577943839Z\",\n      \"endTime\": \"2026-08-25T11:23:57.610020208Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: certified-operators-catalog deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-x7w9rq with version 4.20.35\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000218ae0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-x7w9rq: failed waiting for cluster=\"np-version-upgrade-cluster-x7w9rq\" in resourcegroup=\"rg-np-version-upgrade-x7w9rq-dvf8r82dxzzq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\",\n      \"name\": \"96c551ed-bf22-4ca0-a2f9-b01ce2e4cd5e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:57.577943839Z\",\n      \"endTime\": \"2026-08-25T11:23:57.610020208Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: certified-operators-catalog deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"93f75a595fe834ff90d354323b1795c64c6aa1fffff5420185d0cc08b6045965","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ef607a572d4c53f3a2663eb448d653e968a80b7e9c4d5dd17a7cbefc30c64148","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, cluster-node-tuning-operator, cluster-policy-controller, ingress-operator, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008526a0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-x8svbm7w8gvv\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37ed00b3-c155-4497-b144-33a7a3897320\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37ed00b3-c155-4497-b144-33a7a3897320\",\n      \"name\": \"37ed00b3-c155-4497-b144-33a7a3897320\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:28.292639561Z\",\n      \"endTime\": \"2026-08-25T11:21:35.949263958Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, cluster-network-operator, cluster-node-tuning-operator, cluster-policy-controller, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008526a0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-x8svbm7w8gvv\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37ed00b3-c155-4497-b144-33a7a3897320\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37ed00b3-c155-4497-b144-33a7a3897320\",\n      \"name\": \"37ed00b3-c155-4497-b144-33a7a3897320\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:28.292639561Z\",\n      \"endTime\": \"2026-08-25T11:21:35.949263958Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, cluster-network-operator, cluster-node-tuning-operator, cluster-policy-controller, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 2 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"5c82c50cdd5f477dc9bec1b74ec5428943202baab54c500d7530303ea24f62fe","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"11980404af14f04ae347cf052e220be11d0d2393fbd6aa0e5b7916303d89b95b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-k6tv8vdvt6xx/cluster-candidate-4-22-zlxjzv should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018feb00\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-zlxjzv: failed waiting for cluster=\"cluster-candidate-4-22-zlxjzv\" in resourcegroup=\"rg-candidate-4-22-k6tv8vdvt6xx\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9dd2d4-422b-44d3-9712-ae568d04d7dd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9dd2d4-422b-44d3-9712-ae568d04d7dd\",\n      \"name\": \"7c9dd2d4-422b-44d3-9712-ae568d04d7dd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:38.96838764Z\",\n      \"endTime\": \"2026-08-25T11:23:43.158826266Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, ingress-operator, dns-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-k6tv8vdvt6xx/cluster-candidate-4-22-zlxjzv should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018feb00\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-zlxjzv: failed waiting for cluster=\"cluster-candidate-4-22-zlxjzv\" in resourcegroup=\"rg-candidate-4-22-k6tv8vdvt6xx\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9dd2d4-422b-44d3-9712-ae568d04d7dd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9dd2d4-422b-44d3-9712-ae568d04d7dd\",\n      \"name\": \"7c9dd2d4-422b-44d3-9712-ae568d04d7dd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:38.96838764Z\",\n      \"endTime\": \"2026-08-25T11:23:43.158826266Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, csi-snapshot-controller-operator, ingress-operator, dns-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"2bdb39b170bc1309151a9f820a6a2455edec3be17d92a609195106d19a9d06dc","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"172dca1d402dce2af7dedfe8fcbd440e4917e443080839ac33b06bea5d61d387","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008d98e0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-lnqr4972dtpb\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f003e8b0-f989-4c38-8deb-37f35273e17d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f003e8b0-f989-4c38-8deb-37f35273e17d\",\n      \"name\": \"f003e8b0-f989-4c38-8deb-37f35273e17d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:18:20.791598979Z\",\n      \"endTime\": \"2026-08-25T19:37:20.738606915Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, csi-snapshot-controller-operator, cluster-network-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008d98e0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-lnqr4972dtpb\" to finish creating: GET https://rp.j5965184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f003e8b0-f989-4c38-8deb-37f35273e17d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f003e8b0-f989-4c38-8deb-37f35273e17d\",\n      \"name\": \"f003e8b0-f989-4c38-8deb-37f35273e17d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:18:20.791598979Z\",\n      \"endTime\": \"2026-08-25T19:37:20.738606915Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, csi-snapshot-controller-operator, cluster-network-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314781535965184","occurred_at":"2026-08-25T18:14:42Z","signature_id":"5648256268d040d66790c83ce020dd81edbe55daec10ac344f210eb0a2221d9a","pr_number":6554,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"74426a739e4f86dd5411cdf4cc53f0d913b028f0d225517b981efa72c5de466f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, dns-operator, ingress-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b325c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hcdb8hh7hcl9\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a36ee1da-d072-4335-bca2-f17838bce647\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a36ee1da-d072-4335-bca2-f17838bce647\",\n      \"name\": \"a36ee1da-d072-4335-bca2-f17838bce647\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:49.267992354Z\",\n      \"endTime\": \"2026-08-25T11:23:53.017585492Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-network-operator, ingress-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b325c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hcdb8hh7hcl9\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a36ee1da-d072-4335-bca2-f17838bce647\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a36ee1da-d072-4335-bca2-f17838bce647\",\n      \"name\": \"a36ee1da-d072-4335-bca2-f17838bce647\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:49.267992354Z\",\n      \"endTime\": \"2026-08-25T11:23:53.017585492Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-network-operator, ingress-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"a52aed8bdf10bcdf14463a386f5de9b79532d4654760f91890403f2b46a1a5f7","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e9bb62f46de99c0acdbb4abf9f88a23afd79f058d2b6721fedb3e87d7c10a712","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, dns-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-jmx49s\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0019b2760\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-jmx49s: failed waiting for cluster=\"cluster-zstream-5-0-jmx49s\" in resourcegroup=\"rg-zstream-upgrade-5-0-j2n79nx59vfm\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\",\n      \"name\": \"f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:27.608916079Z\",\n      \"endTime\": \"2026-08-25T11:21:32.925137069Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-jmx49s\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0019b2760\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-jmx49s: failed waiting for cluster=\"cluster-zstream-5-0-jmx49s\" in resourcegroup=\"rg-zstream-upgrade-5-0-j2n79nx59vfm\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\",\n      \"name\": \"f2fd936b-5571-4d4a-8f06-2d24bc0c10ca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:27.608916079Z\",\n      \"endTime\": \"2026-08-25T11:21:32.925137069Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, ingress-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"d33c4e76ff9a43fcb2716c265a586de405e0886f83620fe7fc6fe9c586dabbef","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7d7f14514a88680ff6615f00bb81c0bec2c6dadd1b601567325ec72490caf5ca","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e0a6c0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-9mf2mfq4r2d9\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8a52f10-9e35-4b81-8bed-515ddc39f2b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8a52f10-9e35-4b81-8bed-515ddc39f2b4\",\n      \"name\": \"d8a52f10-9e35-4b81-8bed-515ddc39f2b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:12.867884764Z\",\n      \"endTime\": \"2026-08-25T11:22:17.332591987Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e0a6c0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-9mf2mfq4r2d9\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8a52f10-9e35-4b81-8bed-515ddc39f2b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d8a52f10-9e35-4b81-8bed-515ddc39f2b4\",\n      \"name\": \"d8a52f10-9e35-4b81-8bed-515ddc39f2b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:12.867884764Z\",\n      \"endTime\": \"2026-08-25T11:22:17.332591987Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"4bd8257d88ff066db3ad8c08e1600658f460390b8e6e515cc65d3100aa511a22","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"247cc84147e4574f2635a831e66f7a46a1eb594df690c18944f1a846c19ad8ca","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-policy-controller, cluster-storage-operator, dns-operator, ingress-operator, redhat-marketplace-catalog, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018da440\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qd5fftk5xqnx\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c61f442-7e7a-4287-a096-d40849d2499b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c61f442-7e7a-4287-a096-d40849d2499b\",\n      \"name\": \"3c61f442-7e7a-4287-a096-d40849d2499b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:37.077702612Z\",\n      \"endTime\": \"2026-08-25T16:40:38.597952289Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, ingress-operator, dns-operator, cluster-image-registry-operator, redhat-operators-catalog, redhat-marketplace-catalog, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018da440\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qd5fftk5xqnx\" to finish creating: GET https://rp.j0672512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c61f442-7e7a-4287-a096-d40849d2499b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c61f442-7e7a-4287-a096-d40849d2499b\",\n      \"name\": \"3c61f442-7e7a-4287-a096-d40849d2499b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:21:37.077702612Z\",\n      \"endTime\": \"2026-08-25T16:40:38.597952289Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, ingress-operator, dns-operator, cluster-image-registry-operator, redhat-operators-catalog, redhat-marketplace-catalog, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269870740672512","occurred_at":"2026-08-25T15:16:24Z","signature_id":"390df46832f7606456e52cdf4a307b68f19cb093f5c95774b04156c02106bca8","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8a2bad4325413455e9a17bdb9d76f2dce3a57579df59da3db41f3afd4283d135","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, csi-snapshot-controller-operator, ingress-operator, redhat-marketplace-catalog, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T18:39:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00134aa60\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5ctgfw4z77n6\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/375f0981-14a8-4380-a7e8-1e4162f764ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/375f0981-14a8-4380-a7e8-1e4162f764ae\",\n      \"name\": \"375f0981-14a8-4380-a7e8-1e4162f764ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:29:02.748319627Z\",\n      \"endTime\": \"2026-08-19T19:48:09.81648964Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, redhat-operators-catalog, csi-snapshot-controller-operator, ingress-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00134aa60\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5ctgfw4z77n6\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/375f0981-14a8-4380-a7e8-1e4162f764ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/375f0981-14a8-4380-a7e8-1e4162f764ae\",\n      \"name\": \"375f0981-14a8-4380-a7e8-1e4162f764ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:29:02.748319627Z\",\n      \"endTime\": \"2026-08-19T19:48:09.81648964Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, redhat-operators-catalog, csi-snapshot-controller-operator, ingress-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"15c8cd6151771fba631053e4c516e39b3df9fdc4db1e62da2330bcefb8214b01","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"279b8dc059ffca13ba2e7632bee0bb38b6b6e79fa3da4d820708453f05c04b29","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, dns-operator, ignition-server-proxy, kube-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00059c3e0\u003e: \n    failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-24c5bjzcljlb\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b21aa181-e22f-4f73-9442-34e06ecf2205\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b21aa181-e22f-4f73-9442-34e06ecf2205\",\n      \"name\": \"b21aa181-e22f-4f73-9442-34e06ecf2205\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:55.381643689Z\",\n      \"endTime\": \"2026-08-25T11:23:57.677492232Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00059c3e0\u003e: \n    failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-24c5bjzcljlb\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b21aa181-e22f-4f73-9442-34e06ecf2205\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b21aa181-e22f-4f73-9442-34e06ecf2205\",\n      \"name\": \"b21aa181-e22f-4f73-9442-34e06ecf2205\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:55.381643689Z\",\n      \"endTime\": \"2026-08-25T11:23:57.677492232Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"41cf799b841c41ca021361bf55e60796cb2a72779166c6626d1c1aedefd1fe20","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"65963d58371f4feaf1ead13b45a131ed3c4d34cd31f41683e65892b6e7918355","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, dns-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-4xrqktzlmtt5/cluster-candidate-4-22-rc2wqs should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0019065e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-rc2wqs: failed waiting for cluster=\"cluster-candidate-4-22-rc2wqs\" in resourcegroup=\"rg-candidate-4-22-4xrqktzlmtt5\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af9b261c-a44f-4aca-93ac-ca9eb2668f2a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af9b261c-a44f-4aca-93ac-ca9eb2668f2a\",\n      \"name\": \"af9b261c-a44f-4aca-93ac-ca9eb2668f2a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:18:41.204224191Z\",\n      \"endTime\": \"2026-08-25T19:37:43.979682955Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-4xrqktzlmtt5/cluster-candidate-4-22-rc2wqs should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0019065e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-rc2wqs: failed waiting for cluster=\"cluster-candidate-4-22-rc2wqs\" in resourcegroup=\"rg-candidate-4-22-4xrqktzlmtt5\" to finish creating: GET https://rp.j7285632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af9b261c-a44f-4aca-93ac-ca9eb2668f2a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af9b261c-a44f-4aca-93ac-ca9eb2668f2a\",\n      \"name\": \"af9b261c-a44f-4aca-93ac-ca9eb2668f2a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:18:41.204224191Z\",\n      \"endTime\": \"2026-08-25T19:37:43.979682955Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092314615177285632","occurred_at":"2026-08-25T18:14:12Z","signature_id":"62dce0e08ee026ebe516bd675e439a69e85ef9b00198fc35594656b8245c2bc5","pr_number":6583,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"951de8af83bafe4594b1180a34b1f825907eff4cdbe9d6ea0e8637301c989320","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, dns-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001372260\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-b748z7nmqrc2\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4e67768d-4fd8-4d6f-b652-cd3c57cfd546\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4e67768d-4fd8-4d6f-b652-cd3c57cfd546\",\n      \"name\": \"4e67768d-4fd8-4d6f-b652-cd3c57cfd546\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:48.323976798Z\",\n      \"endTime\": \"2026-08-25T11:21:53.061758244Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, ingress-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001372260\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-b748z7nmqrc2\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4e67768d-4fd8-4d6f-b652-cd3c57cfd546\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4e67768d-4fd8-4d6f-b652-cd3c57cfd546\",\n      \"name\": \"4e67768d-4fd8-4d6f-b652-cd3c57cfd546\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:48.323976798Z\",\n      \"endTime\": \"2026-08-25T11:21:53.061758244Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, ingress-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"dc93c011273628c3be4689097fa54bae4867b676ee6f408953d5289c92e59a88","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9ac40957e0d202cc6756c5cbbe3d0ef3ac7730a3425dfc91631cac2e91ff702d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-xk9vbv\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001000ca0\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-xk9vbv: failed waiting for cluster=\"cluster-zstream-5-0-xk9vbv\" in resourcegroup=\"rg-zstream-upgrade-5-0-9zdkrc5gfnqq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9f8e41-e479-4f6e-9ade-6570233d1d04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9f8e41-e479-4f6e-9ade-6570233d1d04\",\n      \"name\": \"7c9f8e41-e479-4f6e-9ade-6570233d1d04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:05.893167006Z\",\n      \"endTime\": \"2026-08-25T11:24:07.716194982Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-xk9vbv\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001000ca0\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-xk9vbv: failed waiting for cluster=\"cluster-zstream-5-0-xk9vbv\" in resourcegroup=\"rg-zstream-upgrade-5-0-9zdkrc5gfnqq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9f8e41-e479-4f6e-9ade-6570233d1d04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c9f8e41-e479-4f6e-9ade-6570233d1d04\",\n      \"name\": \"7c9f8e41-e479-4f6e-9ade-6570233d1d04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:05.893167006Z\",\n      \"endTime\": \"2026-08-25T11:24:07.716194982Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"e70ccddbda0336cf522314d54bd2bb07c61aafcb132de78c1ca60fa8b7fd1d35","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"66285e3570a1d3a91819718028ead75ff34527595051ad0083e72a103f62194a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, ingress-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014868a0\u003e: \n    failed to create HCP cluster np-dg-minor-6lfgc2: failed waiting for cluster=\"np-dg-minor-6lfgc2\" in resourcegroup=\"rg-np-dg-minor-6lfgc2-lwtz9m4rnlh9\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/169a9726-cf09-492c-bf9a-53259660198c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/169a9726-cf09-492c-bf9a-53259660198c\",\n      \"name\": \"169a9726-cf09-492c-bf9a-53259660198c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:53.246951809Z\",\n      \"endTime\": \"2026-08-25T11:20:57.403791086Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014868a0\u003e: \n    failed to create HCP cluster np-dg-minor-6lfgc2: failed waiting for cluster=\"np-dg-minor-6lfgc2\" in resourcegroup=\"rg-np-dg-minor-6lfgc2-lwtz9m4rnlh9\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/169a9726-cf09-492c-bf9a-53259660198c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/169a9726-cf09-492c-bf9a-53259660198c\",\n      \"name\": \"169a9726-cf09-492c-bf9a-53259660198c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:53.246951809Z\",\n      \"endTime\": \"2026-08-25T11:20:57.403791086Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"09dd934fdb303c274c99f6ad4ed42588a7c9f786ef23d8ba1c2d65fa831cb2e2","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a12a1aaebf551a02f7b6b7033999753bb4e7d2b0f9d00ac7c03c67219d973f16","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00049fd20\u003e: \n    failed to create HCP cluster np-downgrade-gwfjf8: failed waiting for cluster=\"np-downgrade-gwfjf8\" in resourcegroup=\"rg-np-downgrade-gwfjf8-7kkgmzwhgxk9\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a2fd7b6-8052-4eb4-a31e-21858150d69e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a2fd7b6-8052-4eb4-a31e-21858150d69e\",\n      \"name\": \"5a2fd7b6-8052-4eb4-a31e-21858150d69e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:40.69418327Z\",\n      \"endTime\": \"2026-08-25T11:23:43.19279554Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, cluster-image-registry-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00049fd20\u003e: \n    failed to create HCP cluster np-downgrade-gwfjf8: failed waiting for cluster=\"np-downgrade-gwfjf8\" in resourcegroup=\"rg-np-downgrade-gwfjf8-7kkgmzwhgxk9\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a2fd7b6-8052-4eb4-a31e-21858150d69e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a2fd7b6-8052-4eb4-a31e-21858150d69e\",\n      \"name\": \"5a2fd7b6-8052-4eb4-a31e-21858150d69e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:40.69418327Z\",\n      \"endTime\": \"2026-08-25T11:23:43.19279554Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, cluster-image-registry-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"660c5eae4bcca0af6b2fe0cfd998feca24db60cd960d79c76beb62534b620fc5","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c4fdcee2df11b558d3b3164768a55e10395e8648b1940e20c553175bae5d91ab","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-t52ww6\" with version 4.22.10 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00183c860\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-t52ww6: failed waiting for cluster=\"cluster-zstream-4-22-t52ww6\" in resourcegroup=\"rg-zstream-upgrade-4-22-rdnc78mtxx5j\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c696222-49a6-4ca9-b706-8e5a57435480\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c696222-49a6-4ca9-b706-8e5a57435480\",\n      \"name\": \"9c696222-49a6-4ca9-b706-8e5a57435480\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:06.248454811Z\",\n      \"endTime\": \"2026-08-25T11:24:07.701920992Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: cluster-image-registry-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-t52ww6\" with version 4.22.10 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00183c860\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-t52ww6: failed waiting for cluster=\"cluster-zstream-4-22-t52ww6\" in resourcegroup=\"rg-zstream-upgrade-4-22-rdnc78mtxx5j\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c696222-49a6-4ca9-b706-8e5a57435480\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c696222-49a6-4ca9-b706-8e5a57435480\",\n      \"name\": \"9c696222-49a6-4ca9-b706-8e5a57435480\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:06.248454811Z\",\n      \"endTime\": \"2026-08-25T11:24:07.701920992Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: cluster-image-registry-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"47245e609d7874f7d25c869a473a1e7e721abc0d258fe07dc19210de937d516d","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"deacd16fd42c584f7532a98cc9b198caf769401900c4d495ba1223ef893fffc3","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, cluster-policy-controller, cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000936480\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-9ddgb764dmsn\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cd78ff4-590b-4624-b139-ffc7ac943f90\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cd78ff4-590b-4624-b139-ffc7ac943f90\",\n      \"name\": \"7cd78ff4-590b-4624-b139-ffc7ac943f90\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:39.015923741Z\",\n      \"endTime\": \"2026-08-25T11:23:43.160352144Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, dns-operator, packageserver, cluster-policy-controller, ingress-operator, olm-operator, cluster-storage-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000936480\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-9ddgb764dmsn\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cd78ff4-590b-4624-b139-ffc7ac943f90\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7cd78ff4-590b-4624-b139-ffc7ac943f90\",\n      \"name\": \"7cd78ff4-590b-4624-b139-ffc7ac943f90\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:39.015923741Z\",\n      \"endTime\": \"2026-08-25T11:23:43.160352144Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, dns-operator, packageserver, cluster-policy-controller, ingress-operator, olm-operator, cluster-storage-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"fe7dfc068c8c4065aa24dac07dd321a5e46b5ffb23c7bef9e08da4d256504475","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a169e99d8eb2639f0a1059675f96bc5d6a3645d7a85053031c7e676b4ad42f09","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, csi-snapshot-controller-operator, dns-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-j4v9fp\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eaa980\u003e: \n    failed to create HCP cluster np-noedge-j4v9fp: failed waiting for cluster=\"np-noedge-j4v9fp\" in resourcegroup=\"rg-np-noedge-j4v9fp-4r7jnknp4pwv\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\",\n      \"name\": \"e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:30.442745031Z\",\n      \"endTime\": \"2026-08-25T11:23:33.077388395Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-j4v9fp\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eaa980\u003e: \n    failed to create HCP cluster np-noedge-j4v9fp: failed waiting for cluster=\"np-noedge-j4v9fp\" in resourcegroup=\"rg-np-noedge-j4v9fp-4r7jnknp4pwv\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\",\n      \"name\": \"e41d1c34-8f1a-4c5a-b4ca-387c7744ff7c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:30.442745031Z\",\n      \"endTime\": \"2026-08-25T11:23:33.077388395Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"bbf6b517c402b8dff8d7ee72ef0aee64132aec114ab4d1d13c39df47394f1e2f","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fc48aeaa99b8367ce8df90e24c1324426e971a1da3ed42097c5345639e1ee2c0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, dns-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-p6zdptkz52r2/cluster-candidate-4-21-8fkhjq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e62640\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-8fkhjq: failed waiting for cluster=\"cluster-candidate-4-21-8fkhjq\" in resourcegroup=\"rg-candidate-4-21-p6zdptkz52r2\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb1c20d-15c0-4b46-a2f5-9b3caa950874\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb1c20d-15c0-4b46-a2f5-9b3caa950874\",\n      \"name\": \"8cb1c20d-15c0-4b46-a2f5-9b3caa950874\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:40.508277835Z\",\n      \"endTime\": \"2026-08-25T19:39:42.488198943Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, dns-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-p6zdptkz52r2/cluster-candidate-4-21-8fkhjq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e62640\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-8fkhjq: failed waiting for cluster=\"cluster-candidate-4-21-8fkhjq\" in resourcegroup=\"rg-candidate-4-21-p6zdptkz52r2\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb1c20d-15c0-4b46-a2f5-9b3caa950874\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8cb1c20d-15c0-4b46-a2f5-9b3caa950874\",\n      \"name\": \"8cb1c20d-15c0-4b46-a2f5-9b3caa950874\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:20:40.508277835Z\",\n      \"endTime\": \"2026-08-25T19:39:42.488198943Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, dns-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"32b84312ecc0412cc4b93a9ef15c32fc382121719b6562881ec5a616688de541","pr_number":6683,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a0371ff83ff8ae672d5d38be2f67ae6f726e275d3db1d866178e427376320fd8","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011325d0\u003e: \n    GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/642ab867-7117-49d1-9205-fcc81b8fcac8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/642ab867-7117-49d1-9205-fcc81b8fcac8\",\n      \"name\": \"642ab867-7117-49d1-9205-fcc81b8fcac8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:23.530391484Z\",\n      \"endTime\": \"2026-08-25T11:22:27.900506106Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011325d0\u003e: \n    GET https://rp.j9005312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/642ab867-7117-49d1-9205-fcc81b8fcac8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/642ab867-7117-49d1-9205-fcc81b8fcac8\",\n      \"name\": \"642ab867-7117-49d1-9205-fcc81b8fcac8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:23.530391484Z\",\n      \"endTime\": \"2026-08-25T11:22:27.900506106Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092196022469005312","occurred_at":"2026-08-25T10:22:57Z","signature_id":"f83b12586d201faafc5d77e61170761a01913b4aa6e23e7374f34e7086cab7fc","pr_number":6670,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fc2f22952e29415c0ffa879b795fd94133860d84ce001832877fc2b42797fd71","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001048a80\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-znbww55lj6vk\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32f30388-3946-451c-abc4-89ec972b44f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32f30388-3946-451c-abc4-89ec972b44f5\",\n      \"name\": \"32f30388-3946-451c-abc4-89ec972b44f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:32.564773496Z\",\n      \"endTime\": \"2026-08-25T11:21:36.023308749Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-network-operator, openshift-controller-manager, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001048a80\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-znbww55lj6vk\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32f30388-3946-451c-abc4-89ec972b44f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/32f30388-3946-451c-abc4-89ec972b44f5\",\n      \"name\": \"32f30388-3946-451c-abc4-89ec972b44f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:32.564773496Z\",\n      \"endTime\": \"2026-08-25T11:21:36.023308749Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-network-operator, openshift-controller-manager, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"59617915775eae13a40addb69d6b99b7fe7f688ae59ce79fcd656898578c9492","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"910e6132d71017042ccb4ad8536b725ec097c3baf39c134fb4939471b7ea12a4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001696280\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-j578jj9q95nl\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd6036ee-637e-4111-981e-76356c491fc6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd6036ee-637e-4111-981e-76356c491fc6\",\n      \"name\": \"dd6036ee-637e-4111-981e-76356c491fc6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:39.42538851Z\",\n      \"endTime\": \"2026-08-25T11:23:43.113756051Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, ingress-operator, olm-operator, catalog-operator, packageserver, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001696280\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-j578jj9q95nl\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd6036ee-637e-4111-981e-76356c491fc6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dd6036ee-637e-4111-981e-76356c491fc6\",\n      \"name\": \"dd6036ee-637e-4111-981e-76356c491fc6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:39.42538851Z\",\n      \"endTime\": \"2026-08-25T11:23:43.113756051Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, openshift-controller-manager, ingress-operator, olm-operator, catalog-operator, packageserver, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"9a4f67edde370308f7181e7ac96b3504189ee39d35011561ed75c7aba37ebe2d","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5f2092ac8b60125a2e24e70258e522230cf6619c6ea4fc2f4a10c6a15fc6e2e8","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e24460\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-cmpnvt9r42hk\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f79a299e-0c13-4f75-b94f-813c143c5081\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f79a299e-0c13-4f75-b94f-813c143c5081\",\n      \"name\": \"f79a299e-0c13-4f75-b94f-813c143c5081\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T10:58:09.116151219Z\",\n      \"endTime\": \"2026-08-25T11:17:17.145229508Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e24460\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-cmpnvt9r42hk\" to finish creating: GET https://rp.j8697984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f79a299e-0c13-4f75-b94f-813c143c5081\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f79a299e-0c13-4f75-b94f-813c143c5081\",\n      \"name\": \"f79a299e-0c13-4f75-b94f-813c143c5081\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T10:58:09.116151219Z\",\n      \"endTime\": \"2026-08-25T11:17:17.145229508Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193118198697984","occurred_at":"2026-08-25T10:02:51Z","signature_id":"a557455b2f15d5267c35100305fbb366e3ce937bce05ea40aa0c0c92aaacdae2","pr_number":6648,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"3c03ff8c6f57d65dc5b13024ef220ab850aed54bb6437f3a0bfad2241929fe27","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator, cluster-policy-controller, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-g4wtj9 with version 4.20.35\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a8a620\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-g4wtj9: failed waiting for cluster=\"np-version-upgrade-cluster-g4wtj9\" in resourcegroup=\"rg-np-version-upgrade-g4wtj9-mrccxbqqnhhc\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d40e3df-681f-4926-943f-226d7903f59e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d40e3df-681f-4926-943f-226d7903f59e\",\n      \"name\": \"7d40e3df-681f-4926-943f-226d7903f59e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:40.789915832Z\",\n      \"endTime\": \"2026-08-25T11:23:43.181882794Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-policy-controller, cluster-node-tuning-operator, azure-cloud-controller-manager, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-g4wtj9 with version 4.20.35\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a8a620\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-g4wtj9: failed waiting for cluster=\"np-version-upgrade-cluster-g4wtj9\" in resourcegroup=\"rg-np-version-upgrade-g4wtj9-mrccxbqqnhhc\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d40e3df-681f-4926-943f-226d7903f59e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d40e3df-681f-4926-943f-226d7903f59e\",\n      \"name\": \"7d40e3df-681f-4926-943f-226d7903f59e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:40.789915832Z\",\n      \"endTime\": \"2026-08-25T11:23:43.181882794Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-policy-controller, cluster-node-tuning-operator, azure-cloud-controller-manager, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"19ac66d7406c2cf544c19dab33599323ce924d5b29029bc616b279fbe8796bf1","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"55a75f995426be3c05299ad4947251679d4a99ba7e77d726aeadaba9bd31ae5d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator, cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, kube-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-kp6cj8gslgn7/cluster-candidate-4-20-2ftmsx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101ebe0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-2ftmsx: failed waiting for cluster=\"cluster-candidate-4-20-2ftmsx\" in resourcegroup=\"rg-candidate-4-20-kp6cj8gslgn7\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ce27661-0cfc-4c0b-a2fe-99358f1c893d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ce27661-0cfc-4c0b-a2fe-99358f1c893d\",\n      \"name\": \"9ce27661-0cfc-4c0b-a2fe-99358f1c893d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:05.525443298Z\",\n      \"endTime\": \"2026-08-25T11:24:07.636192222Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, packageserver, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-kp6cj8gslgn7/cluster-candidate-4-20-2ftmsx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101ebe0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-2ftmsx: failed waiting for cluster=\"cluster-candidate-4-20-2ftmsx\" in resourcegroup=\"rg-candidate-4-20-kp6cj8gslgn7\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ce27661-0cfc-4c0b-a2fe-99358f1c893d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ce27661-0cfc-4c0b-a2fe-99358f1c893d\",\n      \"name\": \"9ce27661-0cfc-4c0b-a2fe-99358f1c893d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:05.525443298Z\",\n      \"endTime\": \"2026-08-25T11:24:07.636192222Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, cluster-network-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, packageserver, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"7311a955c2510af435a3dd4e8bda65a101a0cfae9846f5be659992b940243062","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"75a2ddc5380cf22cdc0f7eee734c0875c6030bb3df880d9c39c49bb360b07d00","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator, ignition-server-proxy, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a56680\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-h8x2lnscjddq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/306db8ae-e454-406c-b2d9-5d6ef9785b20\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/306db8ae-e454-406c-b2d9-5d6ef9785b20\",\n      \"name\": \"306db8ae-e454-406c-b2d9-5d6ef9785b20\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:01.374469415Z\",\n      \"endTime\": \"2026-08-25T11:24:07.731322209Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, openshift-route-controller-manager, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a56680\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-h8x2lnscjddq\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/306db8ae-e454-406c-b2d9-5d6ef9785b20\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/306db8ae-e454-406c-b2d9-5d6ef9785b20\",\n      \"name\": \"306db8ae-e454-406c-b2d9-5d6ef9785b20\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:01.374469415Z\",\n      \"endTime\": \"2026-08-25T11:24:07.731322209Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, openshift-route-controller-manager, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [cluster-node-tuning-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"674600a738c7667770e739f9055e1d79ea50c97b5b16436d10f3b9df4e0297d0","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b67b2954ed80e23d5dbe844cbe69ddf2aee82b77fbb4d9b923a377beae494cff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7a340\u003e: \n    failed to create HCP cluster np-dg-minor-5k4t7m: failed waiting for cluster=\"np-dg-minor-5k4t7m\" in resourcegroup=\"rg-np-dg-minor-5k4t7m-c7rmk8jp25j5\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36c411c9-a482-476f-8c6d-e111f54a5519\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36c411c9-a482-476f-8c6d-e111f54a5519\",\n      \"name\": \"36c411c9-a482-476f-8c6d-e111f54a5519\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:30.5276884Z\",\n      \"endTime\": \"2026-08-25T11:21:35.939284741Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7a340\u003e: \n    failed to create HCP cluster np-dg-minor-5k4t7m: failed waiting for cluster=\"np-dg-minor-5k4t7m\" in resourcegroup=\"rg-np-dg-minor-5k4t7m-c7rmk8jp25j5\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36c411c9-a482-476f-8c6d-e111f54a5519\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/36c411c9-a482-476f-8c6d-e111f54a5519\",\n      \"name\": \"36c411c9-a482-476f-8c6d-e111f54a5519\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:30.5276884Z\",\n      \"endTime\": \"2026-08-25T11:21:35.939284741Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"4b8149b33710d3d2d55931ca7538e774a86bcfc7ef3d236c36f5ffb2ecb5845f","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"01b6bec7cc22825d4ef87b95818e501dbd62c6d195e057bf40cb1fb4d00f77e0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013da220\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-hk9dqp59j9m5\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\",\n      \"name\": \"cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:54.161881545Z\",\n      \"endTime\": \"2026-08-25T11:20:57.408325813Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013da220\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-hk9dqp59j9m5\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\",\n      \"name\": \"cb6081e4-ae2a-4eab-a6f8-42f62611bc3c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:54.161881545Z\",\n      \"endTime\": \"2026-08-25T11:20:57.408325813Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"dbfac9a592c7d3b659769802f66b5c2ea426b38b8a462e72863bc31b516fbcfb","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"daeefc59f58572b03a630f3ab57fcec5e15bddea741d2154444f66cb96ddf3d9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, dns-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001a5060\u003e: \n    failed to create HCP cluster np-dg-minor-gzvdzt: failed waiting for cluster=\"np-dg-minor-gzvdzt\" in resourcegroup=\"rg-np-dg-minor-gzvdzt-zrllgdp7qt99\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b036348-ef9c-491d-be11-832d77741098\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b036348-ef9c-491d-be11-832d77741098\",\n      \"name\": \"5b036348-ef9c-491d-be11-832d77741098\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:21:00.551807818Z\",\n      \"endTime\": \"2026-08-25T14:40:03.659175976Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001a5060\u003e: \n    failed to create HCP cluster np-dg-minor-gzvdzt: failed waiting for cluster=\"np-dg-minor-gzvdzt\" in resourcegroup=\"rg-np-dg-minor-gzvdzt-zrllgdp7qt99\" to finish creating: GET https://rp.j7004544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b036348-ef9c-491d-be11-832d77741098\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b036348-ef9c-491d-be11-832d77741098\",\n      \"name\": \"5b036348-ef9c-491d-be11-832d77741098\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:21:00.551807818Z\",\n      \"endTime\": \"2026-08-25T14:40:03.659175976Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092240885617004544","occurred_at":"2026-08-25T13:21:13Z","signature_id":"268457b3abac70cef4adc34ed78ed5e91defa790eb3f46b44d42d7bf24a4ab3d","pr_number":6584,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"edc445226bca8c15a6801b9e4e1854b6c7f899e847f419d892bdf3d1968a9d97","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator, dns-operator, ignition-server-proxy, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-n44w2df8vj8t/cluster-candidate-4-21-mppmvd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010c4140\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-mppmvd: failed waiting for cluster=\"cluster-candidate-4-21-mppmvd\" in resourcegroup=\"rg-candidate-4-21-n44w2df8vj8t\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\",\n      \"name\": \"5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:52.19951964Z\",\n      \"endTime\": \"2026-08-25T11:20:57.290621961Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, csi-snapshot-controller-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-n44w2df8vj8t/cluster-candidate-4-21-mppmvd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010c4140\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-mppmvd: failed waiting for cluster=\"cluster-candidate-4-21-mppmvd\" in resourcegroup=\"rg-candidate-4-21-n44w2df8vj8t\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\",\n      \"name\": \"5ef974da-bfca-4b86-a5a7-f5f72d3b3ecc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:52.19951964Z\",\n      \"endTime\": \"2026-08-25T11:20:57.290621961Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, csi-snapshot-controller-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"d5ffa576330ff4e403d279cd3e6697f2671ddbf3167fa412a03694b6d7d28267","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"55058ba88f617860ddd7645f53374b164637eda1467294e44a5e2bd25e9411f0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [dns-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000edefa0\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rxw4z4c4mjfj\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d7f39717-bb95-4ba8-b531-5600d9435095\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d7f39717-bb95-4ba8-b531-5600d9435095\",\n      \"name\": \"d7f39717-bb95-4ba8-b531-5600d9435095\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:37.648275001Z\",\n      \"endTime\": \"2026-08-25T11:22:43.091522188Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000edefa0\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rxw4z4c4mjfj\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d7f39717-bb95-4ba8-b531-5600d9435095\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d7f39717-bb95-4ba8-b531-5600d9435095\",\n      \"name\": \"d7f39717-bb95-4ba8-b531-5600d9435095\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:03:37.648275001Z\",\n      \"endTime\": \"2026-08-25T11:22:43.091522188Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, dns-operator; hosted cluster degraded: UnavailableReplicas: [dns-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"1c7a8ccc960575f74b5f63b9d0fee47fa26ccbebde3ae4de7a7bbd10588a7ab3","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8fcfad7ee636c64bcc7ec16c2a0da480eb90b3d648cf1239ba480534695bb74a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010003c0\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-dgghk7xd8fkc\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/813de9e6-c926-4fa8-a38a-500226e3059a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/813de9e6-c926-4fa8-a38a-500226e3059a\",\n      \"name\": \"813de9e6-c926-4fa8-a38a-500226e3059a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:05.708804056Z\",\n      \"endTime\": \"2026-08-25T11:24:07.512424839Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, hosted-cluster-config-operator, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010003c0\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-dgghk7xd8fkc\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/813de9e6-c926-4fa8-a38a-500226e3059a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/813de9e6-c926-4fa8-a38a-500226e3059a\",\n      \"name\": \"813de9e6-c926-4fa8-a38a-500226e3059a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:05.708804056Z\",\n      \"endTime\": \"2026-08-25T11:24:07.512424839Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, hosted-cluster-config-operator, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"18f9ebc5eb73ff84a02d71ec227c141bf94f7ba02eeea0d33eed11a56429d19b","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"691000237ed68ee2bea1eca4972f51b642a5c69ce54dd17b387ef5ef88b01453","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d02900\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-sn9l5gh5sgnq\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/77d8a5d2-c15f-4303-9203-ed2a4dc77c20\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/77d8a5d2-c15f-4303-9203-ed2a4dc77c20\",\n      \"name\": \"77d8a5d2-c15f-4303-9203-ed2a4dc77c20\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:02.408692636Z\",\n      \"endTime\": \"2026-08-25T11:21:07.333560396Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d02900\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-sn9l5gh5sgnq\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/77d8a5d2-c15f-4303-9203-ed2a4dc77c20\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/77d8a5d2-c15f-4303-9203-ed2a4dc77c20\",\n      \"name\": \"77d8a5d2-c15f-4303-9203-ed2a4dc77c20\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:02.408692636Z\",\n      \"endTime\": \"2026-08-25T11:21:07.333560396Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"c5290fb9c57d434bec526c46d513016dd6789c9d085ab04d2a242efeb095b470","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"183ac5e7e1c7f43ad252fd54e6e3fae5ee8421fc47486bdad5d091cff0dbf9a9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ingress-operator, openshift-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-qpdbsl with version 4.21.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e015c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-qpdbsl: failed waiting for cluster=\"np-version-upgrade-cluster-qpdbsl\" in resourcegroup=\"rg-np-version-upgrade-qpdbsl-xxx55fsnfv5d\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\",\n      \"name\": \"35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:07.081081979Z\",\n      \"endTime\": \"2026-08-25T11:24:07.516329781Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-qpdbsl with version 4.21.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e015c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-qpdbsl: failed waiting for cluster=\"np-version-upgrade-cluster-qpdbsl\" in resourcegroup=\"rg-np-version-upgrade-qpdbsl-xxx55fsnfv5d\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\",\n      \"name\": \"35eaac99-6821-4f6d-8ae1-1eb7916b1e9c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:07.081081979Z\",\n      \"endTime\": \"2026-08-25T11:24:07.516329781Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"6f6befa657f715c3b4b4b66b7a559071e94c36c338612799fd64e9fc60a3ceb0","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2256dab0b9b860af42d8774bc9e3bdf3a75814dca1080b715190d3d8ab7a2eb9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ingress-operator, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000706220\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-tdgbq5vc5q77\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\",\n      \"name\": \"e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:45.146044639Z\",\n      \"endTime\": \"2026-08-25T11:21:47.576163129Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, ingress-operator; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000706220\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-tdgbq5vc5q77\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\",\n      \"name\": \"e298fc5b-ea25-4c91-afe3-b4998ff1e5f2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:45.146044639Z\",\n      \"endTime\": \"2026-08-25T11:21:47.576163129Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, ingress-operator; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"80ba9276ccee66ce5f15246d1c46ed27a227c963837a0639556a2c98e2273b4b","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5d1725423ee4b212bccb0d13858a78dd47769db5d794223ea2e26d6df7c4f5ad","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ingress-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-rc5jrpz77lgk/cluster-candidate-5-0-k8xlxd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001060a40\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-k8xlxd: failed waiting for cluster=\"cluster-candidate-5-0-k8xlxd\" in resourcegroup=\"rg-candidate-5-0-rc5jrpz77lgk\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/223cfa5a-75b0-4cb9-b839-dc5100d41095\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/223cfa5a-75b0-4cb9-b839-dc5100d41095\",\n      \"name\": \"223cfa5a-75b0-4cb9-b839-dc5100d41095\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:32.648244013Z\",\n      \"endTime\": \"2026-08-25T11:20:37.341589368Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-rc5jrpz77lgk/cluster-candidate-5-0-k8xlxd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001060a40\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-k8xlxd: failed waiting for cluster=\"cluster-candidate-5-0-k8xlxd\" in resourcegroup=\"rg-candidate-5-0-rc5jrpz77lgk\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/223cfa5a-75b0-4cb9-b839-dc5100d41095\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/223cfa5a-75b0-4cb9-b839-dc5100d41095\",\n      \"name\": \"223cfa5a-75b0-4cb9-b839-dc5100d41095\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:32.648244013Z\",\n      \"endTime\": \"2026-08-25T11:20:37.341589368Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator; hosted cluster degraded: UnavailableReplicas: [ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"3ab9e7c7d4f2d79e65f9b8a56c90ee66becb35144ac46e78ca458d7f63faa98d","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bb52007d345fecf08b7c3e5921187b32bee560cfda2f40a385c2e2262d097a34","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a1cce0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-994pfg4rbw2z\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\",\n      \"name\": \"f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:18.051164248Z\",\n      \"endTime\": \"2026-08-25T11:23:23.080049738Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, konnectivity-agent, catalog-operator, control-plane-pki-operator, openshift-apiserver, ignition-server-proxy, openshift-route-controller-manager, cluster-autoscaler, machine-approver, kube-controller-manager, azure-cloud-controller-manager, redhat-marketplace-catalog, openshift-controller-manager, hosted-cluster-config-operator, olm-operator, cluster-version-operator, cluster-policy-controller, cluster-storage-operator, dns-operator, cluster-image-registry-operator, olm-collect-profiles, certified-operators-catalog, cluster-node-tuning-operator, ingress-operator, packageserver, redhat-operators-catalog, community-operators-catalog, kube-scheduler, ignition-server, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a1cce0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-994pfg4rbw2z\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\",\n      \"name\": \"f85cfac1-f7cf-4ded-93fb-a1cd604c6c26\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:18.051164248Z\",\n      \"endTime\": \"2026-08-25T11:23:23.080049738Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, konnectivity-agent, catalog-operator, control-plane-pki-operator, openshift-apiserver, ignition-server-proxy, openshift-route-controller-manager, cluster-autoscaler, machine-approver, kube-controller-manager, azure-cloud-controller-manager, redhat-marketplace-catalog, openshift-controller-manager, hosted-cluster-config-operator, olm-operator, cluster-version-operator, cluster-policy-controller, cluster-storage-operator, dns-operator, cluster-image-registry-operator, olm-collect-profiles, certified-operators-catalog, cluster-node-tuning-operator, ingress-operator, packageserver, redhat-operators-catalog, community-operators-catalog, kube-scheduler, ignition-server, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"1ccba13814040aca0b43b4df7ff1901195302261df9120296071d4873aa80760","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4b3c5c6b82d824a461fdf0a9ceb99cba2cdf827bff592ead991249815b65b39a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001350960\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-44brmcrqlwhd\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc250653-c8e3-42dd-bdb2-80628cccd921\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc250653-c8e3-42dd-bdb2-80628cccd921\",\n      \"name\": \"bc250653-c8e3-42dd-bdb2-80628cccd921\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:24:55.65138247Z\",\n      \"endTime\": \"2026-08-25T16:43:58.821172119Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, redhat-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001350960\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-44brmcrqlwhd\" to finish creating: GET https://rp.j0599680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc250653-c8e3-42dd-bdb2-80628cccd921\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc250653-c8e3-42dd-bdb2-80628cccd921\",\n      \"name\": \"bc250653-c8e3-42dd-bdb2-80628cccd921\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:24:55.65138247Z\",\n      \"endTime\": \"2026-08-25T16:43:58.821172119Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, redhat-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092269882870599680","occurred_at":"2026-08-25T15:16:27Z","signature_id":"a997df37747bd9ae4adc6c1d01f9efa7c9eeb6284eec6b1bc5d69624dc28ac10","pr_number":4566,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f1744105155154513ebbf4bd7117f8888fd7e8a5f221414547d7257b1ee3bbdd","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [openshift-controller-manager, openshift-route-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e26600\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-g94rgnmwffff\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00f455ba-7a0f-4042-81b1-6b4e1d717422\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00f455ba-7a0f-4042-81b1-6b4e1d717422\",\n      \"name\": \"00f455ba-7a0f-4042-81b1-6b4e1d717422\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:28.361104Z\",\n      \"endTime\": \"2026-08-25T11:21:33.042291432Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e26600\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-g94rgnmwffff\" to finish creating: GET https://rp.j5488384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00f455ba-7a0f-4042-81b1-6b4e1d717422\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00f455ba-7a0f-4042-81b1-6b4e1d717422\",\n      \"name\": \"00f455ba-7a0f-4042-81b1-6b4e1d717422\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:28.361104Z\",\n      \"endTime\": \"2026-08-25T11:21:33.042291432Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, openshift-route-controller-manager; hosted cluster degraded: UnavailableReplicas: [openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192185385488384","occurred_at":"2026-08-25T10:02:45Z","signature_id":"8306a5db6f3a5a75eed1ac89929790d5c6dfaf67eb9701d7ccbd095c361aa3a6","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f59d41db8c7110a2a66df7a04a929104d4f74b17a121064979556d85ac966c14","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [openshift-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000976d60\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sp25f888spq4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3198a15-7882-4985-8fb9-601f276da5ee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3198a15-7882-4985-8fb9-601f276da5ee\",\n      \"name\": \"d3198a15-7882-4985-8fb9-601f276da5ee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:58.802251942Z\",\n      \"endTime\": \"2026-08-25T11:24:07.715404645Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: openshift-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000976d60\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sp25f888spq4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3198a15-7882-4985-8fb9-601f276da5ee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3198a15-7882-4985-8fb9-601f276da5ee\",\n      \"name\": \"d3198a15-7882-4985-8fb9-601f276da5ee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:04:58.802251942Z\",\n      \"endTime\": \"2026-08-25T11:24:07.715404645Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager; hosted cluster degraded: UnavailableReplicas: openshift-controller-manager deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"4bc6fa9b62fea5fb0a886bde62212833e54bd33c1122e0a8839c522e31ede058","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8a4546d1ccb11eb2bd205f13e52f18ff56616b802ad979a8ada123b2e8ac5a07","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d08560\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mx5c2xs4fbj7\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0554f1ec-f6c0-4f0c-869c-8157fe13f06e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0554f1ec-f6c0-4f0c-869c-8157fe13f06e\",\n      \"name\": \"0554f1ec-f6c0-4f0c-869c-8157fe13f06e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:06:26.179895607Z\",\n      \"endTime\": \"2026-08-25T11:25:27.51902962Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d08560\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mx5c2xs4fbj7\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0554f1ec-f6c0-4f0c-869c-8157fe13f06e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0554f1ec-f6c0-4f0c-869c-8157fe13f06e\",\n      \"name\": \"0554f1ec-f6c0-4f0c-869c-8157fe13f06e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:06:26.179895607Z\",\n      \"endTime\": \"2026-08-25T11:25:27.51902962Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"e7d58e0615757ed6b9924d8c23510714583aa7bdf89e2194c35463a6520b17a5","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9917e0e4cec099cb18cbbcab5a48ffe13a11b43e10f38323888849ed9a27dd49","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, openshift-route-controller-manager, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00058a120\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-hbkfwxhrsvb2\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51ad145b-c45f-488c-9b63-9a804b8af221\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51ad145b-c45f-488c-9b63-9a804b8af221\",\n      \"name\": \"51ad145b-c45f-488c-9b63-9a804b8af221\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:15.619518812Z\",\n      \"endTime\": \"2026-08-25T11:24:17.933561304Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00058a120\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-hbkfwxhrsvb2\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51ad145b-c45f-488c-9b63-9a804b8af221\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51ad145b-c45f-488c-9b63-9a804b8af221\",\n      \"name\": \"51ad145b-c45f-488c-9b63-9a804b8af221\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:15.619518812Z\",\n      \"endTime\": \"2026-08-25T11:24:17.933561304Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"d6a56eb98a1093f3a5c49d3962df7ba45a52a81222c930c3ba06aff802dd1a1f","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"442531230d6bcd61fe689ddb6dd1199a972aefc3d6bea05b5c34201c97dcd098","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-storage-operator, csi-snapshot-controller-operator, dns-operator, hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001336360\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4mq79tbwfj5b\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a3c034-245f-49ad-8068-0b454f6ec56c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a3c034-245f-49ad-8068-0b454f6ec56c\",\n      \"name\": \"a6a3c034-245f-49ad-8068-0b454f6ec56c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:18:48.128240196Z\",\n      \"endTime\": \"2026-08-25T19:37:52.295117327Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001336360\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4mq79tbwfj5b\" to finish creating: GET https://rp.j8405376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a3c034-245f-49ad-8068-0b454f6ec56c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a3c034-245f-49ad-8068-0b454f6ec56c\",\n      \"name\": \"a6a3c034-245f-49ad-8068-0b454f6ec56c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T19:18:48.128240196Z\",\n      \"endTime\": \"2026-08-25T19:37:52.295117327Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-storage-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6683/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092316872178405376","occurred_at":"2026-08-25T18:23:10Z","signature_id":"bc247a67bb8f719d23e560a19b4f04763bcfe81ecbf03218468d9c66f357d587","pr_number":6683,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b652815635f16d983d1325dbd7e304560647aed7cf1be6254e8a2a57faf1d4fa","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ef90e0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2zg88lxq9q4w\" to finish creating: GET https://rp.j3558656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\",\n      \"name\": \"b976f9f3-2891-4433-868a-76a8d23de317\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T16:54:08.369328068Z\",\n      \"endTime\": \"2026-08-24T17:13:08.208412685Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ef90e0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2zg88lxq9q4w\" to finish creating: GET https://rp.j3558656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\",\n      \"name\": \"b976f9f3-2891-4433-868a-76a8d23de317\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T16:54:08.369328068Z\",\n      \"endTime\": \"2026-08-24T17:13:08.208412685Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091920407983558656","occurred_at":"2026-08-24T16:06:36Z","signature_id":"fdde03128e739a2f562ab5c3485dae63a0f561b3b4659197733a62ff1ea2a4e8","pr_number":6584,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bebd56cb7971d5cbd2ba0627c63fe9925ffa3a6799fdb1d98a37f3159a2c38aa","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"also_in":["PROD","STG"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00097b680\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wgz7h9k2skjd\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a299a2cd-ef58-4c09-8fa3-3315683149e1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a299a2cd-ef58-4c09-8fa3-3315683149e1\",\n      \"name\": \"a299a2cd-ef58-4c09-8fa3-3315683149e1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:52.19619894Z\",\n      \"endTime\": \"2026-08-25T11:20:57.271055091Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00097b680\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wgz7h9k2skjd\" to finish creating: GET https://rp.j2151936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a299a2cd-ef58-4c09-8fa3-3315683149e1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a299a2cd-ef58-4c09-8fa3-3315683149e1\",\n      \"name\": \"a299a2cd-ef58-4c09-8fa3-3315683149e1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:01:52.19619894Z\",\n      \"endTime\": \"2026-08-25T11:20:57.271055091Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"f45e48a3b1e3da0b3b86c68ae16171bf54eb097156d7da3966f4506df7b9b4dc","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"3007a8035cb9883551a2dd738a6538996a11b3ee85467615faaa28d2c01efb23","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T17:48:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012ac800\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-k2z986tbg59c\" to finish creating: GET https://rp.j8125696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f045696f-4b84-45c9-977c-312bcc37c82d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f045696f-4b84-45c9-977c-312bcc37c82d\",\n      \"name\": \"f045696f-4b84-45c9-977c-312bcc37c82d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T18:37:34.536304881Z\",\n      \"endTime\": \"2026-08-20T18:56:41.61282925Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.x7u1r2j3j5y3g7s.amf4.j8125696.hcp.osadev.cloud:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 2 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012ac800\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-k2z986tbg59c\" to finish creating: GET https://rp.j8125696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f045696f-4b84-45c9-977c-312bcc37c82d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f045696f-4b84-45c9-977c-312bcc37c82d\",\n      \"name\": \"f045696f-4b84-45c9-977c-312bcc37c82d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T18:37:34.536304881Z\",\n      \"endTime\": \"2026-08-20T18:56:41.61282925Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.x7u1r2j3j5y3g7s.amf4.j8125696.hcp.osadev.cloud:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 2 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"1300ae1779ab2d89390be4908e232d8c8bb56305577ccb0f64ff01c7791b7baf","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b460f0e72ca79f88845e6e8bf228b8d0be2539d0f8bce034dfbee5009ef88258","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3e1a0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-nlgtzd8jj8ls\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c6c05ac-d3c3-4c18-b01d-729aef0c992e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c6c05ac-d3c3-4c18-b01d-729aef0c992e\",\n      \"name\": \"5c6c05ac-d3c3-4c18-b01d-729aef0c992e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:29.862453094Z\",\n      \"endTime\": \"2026-08-25T11:21:35.935998302Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3e1a0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-nlgtzd8jj8ls\" to finish creating: GET https://rp.j7671680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c6c05ac-d3c3-4c18-b01d-729aef0c992e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c6c05ac-d3c3-4c18-b01d-729aef0c992e\",\n      \"name\": \"5c6c05ac-d3c3-4c18-b01d-729aef0c992e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:02:29.862453094Z\",\n      \"endTime\": \"2026-08-25T11:21:35.935998302Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193189497671680","occurred_at":"2026-08-25T10:02:57Z","signature_id":"bc8afc40785a7de130f0afb33506490a59b6681bd8c5fec932af8efd346aaf8c","pr_number":6645,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"114cb61901859f036a8d83d4edb4405b6736c8fe33bb88a26c3df9e5b3d78697","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingOnInfrastructureReady: Cluster infrastructure is still provisioning; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-19T22:35:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a84820\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-g742bqj9glvt\" to finish creating: GET https://rp.j4815104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b362542a-ad60-4297-b5da-8467abaa215b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b362542a-ad60-4297-b5da-8467abaa215b\",\n      \"name\": \"b362542a-ad60-4297-b5da-8467abaa215b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:20:10.360400826Z\",\n      \"endTime\": \"2026-08-19T23:39:12.433437262Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingOnInfrastructureReady: Cluster infrastructure is still provisioning; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a84820\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-g742bqj9glvt\" to finish creating: GET https://rp.j4815104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b362542a-ad60-4297-b5da-8467abaa215b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b362542a-ad60-4297-b5da-8467abaa215b\",\n      \"name\": \"b362542a-ad60-4297-b5da-8467abaa215b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:20:10.360400826Z\",\n      \"endTime\": \"2026-08-19T23:39:12.433437262Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingOnInfrastructureReady: Cluster infrastructure is still provisioning; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090206132864815104","occurred_at":"2026-08-19T22:35:50Z","signature_id":"4885cbb4cc6da23b560a2111d8dfc0586a851d3c87fe24afcb3a3acbe0789700","pr_number":6607,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"09f986eeaef9ef03e89acc8ef57bf3aa4f7820de8da84049b2bc414381ff7c7f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ee3660\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-pfd8fc762xff\" to finish creating: GET https://rp.j9836800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9934e2d-ec51-45a3-8cbd-60f4853ece87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9934e2d-ec51-45a3-8cbd-60f4853ece87\",\n      \"name\": \"c9934e2d-ec51-45a3-8cbd-60f4853ece87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:47:46.321950941Z\",\n      \"endTime\": \"2026-08-25T15:06:53.075223567Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ee3660\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-pfd8fc762xff\" to finish creating: GET https://rp.j9836800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9934e2d-ec51-45a3-8cbd-60f4853ece87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9934e2d-ec51-45a3-8cbd-60f4853ece87\",\n      \"name\": \"c9934e2d-ec51-45a3-8cbd-60f4853ece87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T14:47:46.321950941Z\",\n      \"endTime\": \"2026-08-25T15:06:53.075223567Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092251003909836800","occurred_at":"2026-08-25T14:01:22Z","signature_id":"279d843775eb67f9cc4f7628611a62e9cdb3df14488565ed034ba6ce68dad623","pr_number":6630,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"94457f237fe77f556896b3c5dcbbf4f6fa7202d5ac7c0a13c7473270227156e1","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea0340\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-cm65c9f47lvs\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\",\n      \"name\": \"87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:42:45.179574291Z\",\n      \"endTime\": \"2026-08-25T17:01:49.211726949Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea0340\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-cm65c9f47lvs\" to finish creating: GET https://rp.j0166528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\",\n      \"name\": \"87f7df17-c6af-4eeb-b82d-34b9b0fdeac2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T16:42:45.179574291Z\",\n      \"endTime\": \"2026-08-25T17:01:49.211726949Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092279603430166528","occurred_at":"2026-08-25T15:55:04Z","signature_id":"3b0400f6feabfc5e3f875e41b2ad13cd2a30893f4a99364d4e6f721f7666dfca","pr_number":6679,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8172b3b938ddc92e532decd443ed82e33579c57cbdab74ca3567337de4aab992","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d81e60\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-sx5z6bzj9gt4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9107795d-3348-494d-85dd-e9b48aaf131e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9107795d-3348-494d-85dd-e9b48aaf131e\",\n      \"name\": \"9107795d-3348-494d-85dd-e9b48aaf131e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:17.129504958Z\",\n      \"endTime\": \"2026-08-25T11:24:17.573823206Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d81e60\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-sx5z6bzj9gt4\" to finish creating: GET https://rp.j2121984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9107795d-3348-494d-85dd-e9b48aaf131e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9107795d-3348-494d-85dd-e9b48aaf131e\",\n      \"name\": \"9107795d-3348-494d-85dd-e9b48aaf131e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T11:05:17.129504958Z\",\n      \"endTime\": \"2026-08-25T11:24:17.573823206Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092193248222121984","occurred_at":"2026-08-25T10:03:00Z","signature_id":"76e7fe8a79658b5944b75f155ea6ce85c0f606666b8630971299524820b46509","pr_number":6662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"07c5a276cc557395833fa5a9a3dd00ed9d007140f2aa66df2672b994256306c7","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-19T18:39:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001d84120\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-dkq6w72s7qx7\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56c87b3e-1932-4c66-9b4c-72604659dde3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56c87b3e-1932-4c66-9b4c-72604659dde3\",\n      \"name\": \"56c87b3e-1932-4c66-9b4c-72604659dde3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:22.919700021Z\",\n      \"endTime\": \"2026-08-19T19:47:29.851293041Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001d84120\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-dkq6w72s7qx7\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56c87b3e-1932-4c66-9b4c-72604659dde3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56c87b3e-1932-4c66-9b4c-72604659dde3\",\n      \"name\": \"56c87b3e-1932-4c66-9b4c-72604659dde3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:22.919700021Z\",\n      \"endTime\": \"2026-08-19T19:47:29.851293041Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"dccfd57f9593df2be76271b9211656e5ac05b45c64d18576620ed8bc61afd481","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"63b932f574b85083f97c91322b9a6ecdafe1d1b810c184f1491c16896b37be79","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T18:39:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7cce0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-zqvlg88vd2jj\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0c9c39b-413e-43f6-bf72-32d04b43d23e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0c9c39b-413e-43f6-bf72-32d04b43d23e\",\n      \"name\": \"b0c9c39b-413e-43f6-bf72-32d04b43d23e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:40.237300396Z\",\n      \"endTime\": \"2026-08-19T19:47:49.770337967Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: hosted-cluster-config-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7cce0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-zqvlg88vd2jj\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0c9c39b-413e-43f6-bf72-32d04b43d23e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0c9c39b-413e-43f6-bf72-32d04b43d23e\",\n      \"name\": \"b0c9c39b-413e-43f6-bf72-32d04b43d23e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:40.237300396Z\",\n      \"endTime\": \"2026-08-19T19:47:49.770337967Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: hosted-cluster-config-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"6cb42325713d6fee4731d031f5a77e75d73f77401d53264f8af766970b2dfe91","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d750117ee8ebeaf1ff99e5c34b4cf84643ef0acdb6a9d1ab17022db0a4dd4379","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: externalAuths, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-19T14:43:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00149e1f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-4jhgzkdbjlgs\" to finish deleting: GET https://rp.j2752384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38155a97-ba59-4f8d-8be4-28e12f30ac13\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38155a97-ba59-4f8d-8be4-28e12f30ac13\",\n      \"name\": \"38155a97-ba59-4f8d-8be4-28e12f30ac13\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T16:04:26.177378827Z\",\n      \"endTime\": \"2026-08-19T16:28:27.080589493Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ns3hqlu46g55l00aeovpib48f618n still exists (deletion dispatched at 2026-08-19T16:04:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00149e1f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-4jhgzkdbjlgs\" to finish deleting: GET https://rp.j2752384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38155a97-ba59-4f8d-8be4-28e12f30ac13\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38155a97-ba59-4f8d-8be4-28e12f30ac13\",\n      \"name\": \"38155a97-ba59-4f8d-8be4-28e12f30ac13\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T16:04:26.177378827Z\",\n      \"endTime\": \"2026-08-19T16:28:27.080589493Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ns3hqlu46g55l00aeovpib48f618n still exists (deletion dispatched at 2026-08-19T16:04:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087196852752384","occurred_at":"2026-08-19T14:43:14Z","signature_id":"0070970079376a5ff3304016bcd4e59257420a882773ccd09b7df9dc62e41353","pr_number":6608,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ba89a4166305f6c19c8eaadc8056c4375814afedeefcd4ac9c286060c7d5cad1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-10"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-14T20:23:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000ebc2a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-h4ldj6\" in resourcegroup=\"rg-candidate-5-0-xjnldggtrt7m\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\",\n      \"name\": \"fa0102ea-068c-4907-92f2-69579c623cac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:21:51.698922375Z\",\n      \"endTime\": \"2026-08-24T15:45:59.987785659Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm5ibpilvjmdnr77hbqf9t255bon still exists (deletion dispatched at 2026-08-24T15:21:51Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000ebc2a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-h4ldj6\" in resourcegroup=\"rg-candidate-5-0-xjnldggtrt7m\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\",\n      \"name\": \"fa0102ea-068c-4907-92f2-69579c623cac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:21:51.698922375Z\",\n      \"endTime\": \"2026-08-24T15:45:59.987785659Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm5ibpilvjmdnr77hbqf9t255bon still exists (deletion dispatched at 2026-08-24T15:21:51Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"60c407c2c17844ded754b0d5e8dda40f125391008bbbdd669c23fa4348b5b316","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-lpd7ls\" to 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a5c5a0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-lpd7ls\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-nkpr7nplpjkn\" to finish updating: GET https://rp.j8801025.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-lpd7ls\" to 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a5c5a0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-lpd7ls\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-nkpr7nplpjkn\" to finish updating: GET https://rp.j8801025.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a615d984-573b-4196-87e0-68aeb8a371a3\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092191713148801025","occurred_at":"2026-08-25T10:02:43Z","signature_id":"04481477e7547c1ab7c165b4ddca30acd8f9d43ad49b8ba03ce0070a336cbe84","pr_number":6584,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1a135150bda94b29bbfd120583fd4db7aacc9a2d400307335c08f2083d06f63a","failure_pattern":"ERROR: HTTPSConnectionPool(host='login.microsoftonline.com', port=443): Max retries exceeded with url: /XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/oauth2/v2.0/token (Caused by NameResolutionError(\"HTTPSConnection(host='login.microsoftonline.com', port=443): Failed","search_query":"ERROR: HTTPSConnectionPool(host='login.microsoftonline.com', port=443): Max retries exceeded with url: /XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/oauth2/v2.0/token (Caused by NameResolutionError(\\\"HTTPSConnection(host='login.microsoftonline.com', port=443): Failed to resolve 'login.microsoftonline.com' ([Errno -2] Name or service not known)\\\"))","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Server/global/mirror-image","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Server/global/mirror-image","occurrences":1}],"full_error_samples":["time=2026-08-25T05:18:44.021Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e maestro.image.registry \u003cnil\u003e}:{\u003cnil\u003e maestro.image.repository \u003cnil\u003e}@{\u003cnil\u003e maestro.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-08-25T05:19:25.373Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org registry.build13.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nERROR: HTTPSConnectionPool(host='login.microsoftonline.com', port=443): Max retries exceeded with url: /XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/oauth2/v2.0/token (Caused by NameResolutionError(\\\"HTTPSConnection(host='login.microsoftonline.com', port=443): Failed to resolve 'login.microsoftonline.com' ([Errno -2] Name or service not known)\\\"))\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6667/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092109160337379328","occurred_at":"2026-08-25T04:37:46Z","signature_id":"bfc9e6d451ec941871c7fd17bc88e4377f14864e420626a1eb125b993b91f4d2","pr_number":6667,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"955f2e0940a6e134701e99a0c3ef85d6b5374445ed9f53e16789c85a8fd4d2bc","failure_pattern":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.","search_query":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-08-25T10:31:54.607Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 description=\"Step swift-vnet\\n  Kind: Shell\\n  Command: ./swift-vnet.sh\\n\"\ntime=2026-08-25T10:31:55.356Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: Failed to load or parse file /tmp/azure-cli-config-3697365572/azureProfile.json. It will be overridden by default settings.\\nERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092192723133009920","occurred_at":"2026-08-25T10:03:11Z","signature_id":"561505d61b3a52ebd516a8399919b0bdc92e7d97b736ef08c12b540f9f4f583c","pr_number":6583,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"67620228b260ef66127b0b69960de12fe2eddbeb199df66329f98d7615a3df8a","failure_pattern":"HCP Recovery should recover an HCP cluster with control plane rollback after nodepool and control plane upgrade on 4.20 stable: Timed out after \u003cduration\u003es.","search_query":"Timed out after 900.001s.","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with control plane rollback after nodepool and control plane upgrade on 4.20 stable","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T23:32:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with control plane rollback after nodepool and control plane upgrade on 4.20 stable","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.001s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.001s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090220408950231040","occurred_at":"2026-08-19T23:32:34Z","signature_id":"5b4be62361d18fc5959a8e5233d7b6439eac67487c82050f8a810538bc1b88e3","pr_number":6613,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bf29a540824c1838b936647fffd1b3e436c2538cb5a221801fa7dbd94f0d3b6c","failure_pattern":"HCP Recovery should recover an HCP cluster with nodepool z-stream downgrade reverted by restore on 4.20 stable: Timed out after \u003cduration\u003es.","search_query":"Timed out after 900.000s.","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with nodepool z-stream downgrade reverted by restore on 4.20 stable","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T23:32:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with nodepool z-stream downgrade reverted by restore on 4.20 stable","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.000s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.000s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090220408950231040","occurred_at":"2026-08-19T23:32:34Z","signature_id":"6c98d24c3441b55b9b59099220f661c6b6de4764b8feedcc5f80ea8bd8e2c909","pr_number":6613,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a41887c3af7419c7f9e3647f2a1aff5f2e4dc467b6a5a8d5c25f3d0c7558c548","failure_pattern":"HCP Recovery should recover an HCP cluster with restore from mid-upgrade backup on 4.20 stable: Timed out after \u003cduration\u003es.","search_query":"Timed out after 900.001s.","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with restore from mid-upgrade backup on 4.20 stable","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T23:32:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with restore from mid-upgrade backup on 4.20 stable","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.001s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.001s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090220408950231040","occurred_at":"2026-08-19T23:32:34Z","signature_id":"5b4be62361d18fc5959a8e5233d7b6439eac67487c82050f8a810538bc1b88e3","pr_number":6613,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7db47e556e61c60f515550ddd173fabbc9066fe4dfb5c0a64a6afa95310aa064","failure_pattern":"HCP Recovery should recover an HCP cluster with y-stream upgrade rollback on 4.20 stable: Timed out after \u003cduration\u003es.","search_query":"Timed out after 900.010s.","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with y-stream upgrade rollback on 4.20 stable","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T23:32:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with y-stream upgrade rollback on 4.20 stable","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.010s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.010s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090220408950231040","occurred_at":"2026-08-19T23:32:34Z","signature_id":"db5b8603ae6248568d969b8cf11aa46bed084ef6615bcd15f505703239f03a68","pr_number":6613,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2b0a67e3deae6e622f86a1056f81d35366a09191c38b03a141de005a8d648dc8","failure_pattern":"HCP Recovery should recover an HCP cluster with z-stream upgrade rollback on 4.20 stable: Timed out after \u003cduration\u003es.","search_query":"Timed out after 900.001s.","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with z-stream upgrade rollback on 4.20 stable","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T23:32:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with z-stream upgrade rollback on 4.20 stable","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.001s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.001s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090220408950231040","occurred_at":"2026-08-19T23:32:34Z","signature_id":"5b4be62361d18fc5959a8e5233d7b6439eac67487c82050f8a810538bc1b88e3","pr_number":6613,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"58fc1cbd5bf08c0e9cacc04febc66fa47a6ea8ac19d3acc5b10cdd7a31b22169","failure_pattern":"RevokeCredentialsAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-w9tp85\" in resourcegroup=\"admin-credential-lifecycle-test-xprwv7fr5n4c\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-w9tp85 in resource group admin-credential-lifecycle-test-xprwv7fr5n4c, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-19T19:44:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:253]: failed to revoke admin credentials for cluster \"admin-cred-lifecycle-w9tp85\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009a6ab0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-w9tp85\" in resourcegroup=\"admin-credential-lifecycle-test-xprwv7fr5n4c\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-w9tp85 in resource group admin-credential-lifecycle-test-xprwv7fr5n4c, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:253]: failed to revoke admin credentials for cluster \"admin-cred-lifecycle-w9tp85\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009a6ab0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-w9tp85\" in resourcegroup=\"admin-credential-lifecycle-test-xprwv7fr5n4c\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-w9tp85 in resource group admin-credential-lifecycle-test-xprwv7fr5n4c, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6596/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090162950290542592","occurred_at":"2026-08-19T19:44:15Z","signature_id":"ec269b2597fc31d61d2cc332d64658588a04f3f51ea0c63508f0206e3a910359","pr_number":6596,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"923a50890e47da928e11272929c213214b23d4d0575eb0bee74b8d1c72c1cbad","failure_pattern":"Unauthorized","search_query":"Unauthorized","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-20T09:06:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:231]: Timed out after 300.001s.\nExpected success, but got an error:\n    \u003c*errors.StatusError | 0xc00073de00\u003e: \n    Unauthorized\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:231]: Timed out after 300.001s.\nExpected success, but got an error:\n    \u003c*errors.StatusError | 0xc00073de00\u003e: \n    Unauthorized\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"31da737492c14a291cad1f6e96261dd134ddc62222103253bc9dfb47dd1c39fc","pr_number":6583,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7dd8d2d0eae840e7a1d7a4e2c0c0eff25495785c460c6cce9d24e4e2853ccd76","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e\": EOF","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.fully-private.l5d2.j8125696.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T17:48:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:151]: failed to verify IngressController scope for fully private cluster \"fully-private\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0019a4660\u003e: \n    VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.fully-private.l5d2.j8125696.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": EOF\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.fully-private.l5d2.j8125696.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": EOF\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.fully-private.l5d2.j8125696.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:151]: failed to verify IngressController scope for fully private cluster \"fully-private\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0019a4660\u003e: \n    VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.fully-private.l5d2.j8125696.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": EOF\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.fully-private.l5d2.j8125696.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": EOF\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.fully-private.l5d2.j8125696.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"805fd9d2b8419ec1131d1fd0e38fac4c9a75781f95e4cfc7ecf5abda4cb48ccb","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cf087e7d7ad9a23bbaa2e64e6ff30d2ceb433f2c3d39c4dcad0f061219cdfbdf","failure_pattern":"VerifyBasicAccess failed: failed to list services: Get \"\u003curl\u003e","search_query":"VerifyBasicAccess failed: failed to list services: Get \"https://api","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T17:02:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:99]: failed to verify HCP cluster \"cilium-cl\" is available\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001057158\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.cilium-cl.vi5q.j7223552.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.cilium-cl.vi5q.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:39964-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.cilium-cl.vi5q.j7223552.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.cilium-cl.vi5q.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:39964-\u003e172.30.0.10:53: i/o timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:99]: failed to verify HCP cluster \"cilium-cl\" is available\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001057158\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.cilium-cl.vi5q.j7223552.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.cilium-cl.vi5q.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:39964-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.cilium-cl.vi5q.j7223552.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.cilium-cl.vi5q.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:39964-\u003e172.30.0.10:53: i/o timeout\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090484744067223552","occurred_at":"2026-08-20T17:02:48Z","signature_id":"15ec012c60cd73ddc2be8d5694c34d40f04a7843ebb60a0c6eac3595d1493de1","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e43918bf51405cc433b6fba69bdc1ea9b482230892e03b1d2b08eabffb49ed22","failure_pattern":"VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:108]: failed to verify image registry is disabled on cluster disabled-image-registry-hcp-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc0012d80d8\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n    VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:108]: failed to verify image registry is disabled on cluster disabled-image-registry-hcp-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc0012d80d8\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n    VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": dial tcp: lookup api.m4a3a4d0t8e6l6f.ayq6.j7984128.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.104.78:33481-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092096126327984128","occurred_at":"2026-08-25T03:45:56Z","signature_id":"4654f560dff0807befbb619dbe35d2726f4f20abdaaa079b608640db38766779","pr_number":6583,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"56538debc7c282fbaf42751c4c2a45538d87c5287f64737beff9b071aa52b44a","failure_pattern":"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"\u003curl\u003e\": EOF","search_query":"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.full-priv-kv.c2re.j8125696.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": EOF","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS, private ingress, and private KeyVault and verify all are operational","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T17:48:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS, private ingress, and private KeyVault and verify all are operational","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:182]: failed to verify IngressController scope for fully private cluster with KV \"full-priv-kv\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001c89458\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.full-priv-kv.c2re.j8125696.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": EOF\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.full-priv-kv.c2re.j8125696.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF\n    VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.full-priv-kv.c2re.j8125696.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": EOF\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:182]: failed to verify IngressController scope for fully private cluster with KV \"full-priv-kv\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001c89458\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.full-priv-kv.c2re.j8125696.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": EOF\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.full-priv-kv.c2re.j8125696.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF\n    VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.full-priv-kv.c2re.j8125696.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": EOF\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"2c097d2bfcadce600df666a735b77e9a88be50134175d8a10dfc3aac6697ccec","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"21a20c3f2a410873d47baf5997de12bdfb2c72e1a9838fd77eb6baa1c45a7e35","failure_pattern":"VerifyNodesReady failed: can't list nodes in the cluster: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyNodesReady failed: can't list nodes in the cluster: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/api/v1/nodes\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T06:00:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:332]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002d88cd8\u003e: \n    VerifyCiliumOperational failed: failed to get cilium namespace: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/api/v1/namespaces/kube-system\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout\n    VerifyNodesReady failed: can't list nodes in the cluster: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/api/v1/nodes\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:332]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002d88cd8\u003e: \n    VerifyCiliumOperational failed: failed to get cilium namespace: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/api/v1/namespaces/kube-system\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout\n    VerifyNodesReady failed: can't list nodes in the cluster: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/api/v1/nodes\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089955526287822848","occurred_at":"2026-08-19T06:00:01Z","signature_id":"cb57fccbabef48381df2877e8173bbe75ee75f2910390c74e6f7c13befe3aa4b","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b0f6b13b4a76d7562287bf1face4f5d9fb1b45d4ebf0047290b8a707a05354ee","failure_pattern":"VerifyNodesReady failed: there are not ready nodes: [\u003cnode\u003e]","search_query":"VerifyNodesReady failed: there are not ready nodes: [agg-cluster-agg-np-a-65ddj-wssb4]","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-19T20:53:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:332]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002888ee8\u003e: \n    VerifyNodesReady failed: there are not ready nodes: [agg-cluster-agg-np-a-65ddj-wssb4]\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:332]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002888ee8\u003e: \n    VerifyNodesReady failed: there are not ready nodes: [agg-cluster-agg-np-a-65ddj-wssb4]\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090180258316685312","occurred_at":"2026-08-19T20:53:01Z","signature_id":"7f0eeab00948fd70daf7cbbbde40ef16a2d70b7a0858feb589b98705b5a99b5e","pr_number":6614,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T15:19:37Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001626078\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001626078\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090096353462980608","occurred_at":"2026-08-19T15:19:37Z","signature_id":"67b3ef07ae961ffa750c120255e7902812bbf6a12a0a0d8076d36f33d9a552e8","pr_number":6547,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"745254a66491e8b14022dbd080dc776f7553c02a04c9735ae36d5e5593508f47","failure_pattern":"Yet, Consistently failed because the matcher was not satisfied:","search_query":"Yet, Consistently failed because the matcher was not satisfied:","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T21:45:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:199]: Failed after 300.754s.\nno schedule should execute new backups while paused\nThe function passed to Consistently returned the following error:\n    \u003c*fmt.wrapError | 0xc0012320a0\u003e: \n    failed to send request: Get \"https://admin.j9896192.hcpsvc.osadev.cloud/admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pause-bkp-e2e-qtdq4l5gfbn7/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/pause-bkp-cluster/backupschedules\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\n    ...\nAt one point, however, the function did return successfully.\nYet, Consistently failed because the matcher was not satisfied:\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:199]: Failed after 300.754s.\nno schedule should execute new backups while paused\nThe function passed to Consistently returned the following error:\n    \u003c*fmt.wrapError | 0xc0012320a0\u003e: \n    failed to send request: Get \"https://admin.j9896192.hcpsvc.osadev.cloud/admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pause-bkp-e2e-qtdq4l5gfbn7/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/pause-bkp-cluster/backupschedules\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\n    ...\nAt one point, however, the function did return successfully.\nYet, Consistently failed because the matcher was not satisfied:\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6633/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090555886949896192","occurred_at":"2026-08-20T21:45:38Z","signature_id":"8a333d18823bea22f2d701654aebdd23f71e5242d571b9bb55c50b31b08ad8b0","pr_number":6633,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"2c8d18cdb2b7dd33e8c21a8b6a9a4ce7f6faa5fa98e5020989e6c4479385b197","failure_pattern":"Yet, Eventually failed because the matcher was not satisfied:","search_query":"Yet, Eventually failed because the matcher was not satisfied:","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with a ConfigMap on 4.22 stable","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T23:32:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with a ConfigMap on 4.22 stable","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:287]: Told to stop trying after 30.115s.\nrestore permanently failed: lastCondition=, phase=\nAt one point, however, the function did return successfully.\nYet, Eventually failed because the matcher was not satisfied:\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:287]: Told to stop trying after 30.115s.\nrestore permanently failed: lastCondition=, phase=\nAt one point, however, the function did return successfully.\nYet, Eventually failed because the matcher was not satisfied:\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090220408950231040","occurred_at":"2026-08-19T23:32:34Z","signature_id":"8eed1d93c78e7ad13c568987723ce9aa239dc333646b033e1b574892fc1aa6d4","pr_number":6613,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"429fcab9de847861bc73d647c851f740bbfe30154630193b6c0f1c86d0c6c3c8","failure_pattern":"alert [hcp] KubeDeploymentRolloutStuck fired","search_query":"alert [hcp] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":12,"trend_counts":[3,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":48,"prior_last_seen_at":"2026-08-19T14:06:51Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 22 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:35:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h/capi-provider is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:35:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h/capi-provider is not progressing for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:36:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f/capi-provider is not progressing for longer than 30 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:36:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s/capi-provider is not progressing for longer than 30 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:36:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster/capi-provider is not progressing for longer than 30 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:36:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e/capi-provider is not progressing for longer than 30 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:36:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e/capi-provider is not progressing for longer than 30 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:36:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f/capi-provider is not progressing for longer than 30 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:36:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s/capi-provider is not progressing for longer than 30 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:36:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster/capi-provider is not progressing for longer than 30 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:38:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress/capi-provider is not progressing for longer than 30 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:38:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d/capi-provider is not progressing for longer than 30 minutes.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:38:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d/capi-provider is not progressing for longer than 30 minutes.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:38:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress/capi-provider is not progressing for longer than 30 minutes.\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:38:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r/capi-provider is not progressing for longer than 30 minutes.\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:38:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r/capi-provider is not progressing for longer than 30 minutes.\nFiring 17:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:40:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster/capi-provider is not progressing for longer than 30 minutes.\nFiring 18:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:40:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster/capi-provider is not progressing for longer than 30 minutes.\nFiring 19:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:40:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster/capi-provider is not progressing for longer than 30 minutes.\nFiring 20:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:40:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster/capi-provider is not progressing for longer than 30 minutes.\nFiring 21:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:59:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q/capi-provider is not progressing for longer than 30 minutes.\nFiring 22:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:00:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q/capi-provider is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"51e0c474703df02d6ae5c94d0202e9b971d8b5a564553aa7a9eec0fcfa04c3ac","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9c14268f5937655bde9e381e9f7b23c185ab03a3643c5982db2abac6823bde46","failure_pattern":"alert [hcp] userJourneyKubeApiserverAvailability1h5m fired","search_query":"alert [hcp] userJourneyKubeApiserverAvailability1h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyKubeApiserverAvailability1h5m does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T14:06:51Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyKubeApiserverAvailability1h5m does not fire","occurrences":1}],"full_error_samples":["alert fired 7 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:14:56Z\n  Severity:    Sev3\n  Labels:      _id=\"6051253d-d60f-4b6d-968a-7f03461c4aa0\", alertname=\"userJourneyKubeApiserverAvailability1h5m\", cluster=\"ci01-j0627968-mgmt-1\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", name=\"j0p2h1j4d3t8x5u\", namespace=\"ocm-arohcpci01-2s9n9h5qdf8gb4n07q0k99jcbmc1n5ti\", resource_id=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-skip-minor-gchzdq-ww2xhlmtfnwr/providers/microsoft.redhatopenshift/hcpopenshiftclusters/np-skip-minor-gchzdq\", severity=\"3\", short_window=\"5m\", slo=\"kas-availability\"\n  Description: Resource ID: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-skip-minor-gchzdq-ww2xhlmtfnwr/providers/microsoft.redhatopenshift/hcpopenshiftclusters/np-skip-minor-gchzdq\nManagement Cluster: ci01-j0627968-mgmt-1\nNamespace: ocm-arohcpci01-2s9n9h5qdf8gb4n07q0k99jcbmc1n5ti\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:14:57Z\n  Severity:    Sev3\n  Labels:      _id=\"77131d63-65bc-44c3-962a-dd847abba09a\", alertname=\"userJourneyKubeApiserverAvailability1h5m\", cluster=\"ci01-j0627968-mgmt-1\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", name=\"o3k2b5p1w0y4f7i\", namespace=\"ocm-arohcpci01-2s9n9jldsfibh07akcms9kk0t1s8lje8\", resource_id=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-d9djl4vwvd9d/providers/microsoft.redhatopenshift/hcpopenshiftclusters/np-autoscale-cluster\", severity=\"3\", short_window=\"5m\", slo=\"kas-availability\"\n  Description: Resource ID: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-d9djl4vwvd9d/providers/microsoft.redhatopenshift/hcpopenshiftclusters/np-autoscale-cluster\nManagement Cluster: ci01-j0627968-mgmt-1\nNamespace: ocm-arohcpci01-2s9n9jldsfibh07akcms9kk0t1s8lje8\n\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:14:57Z\n  Severity:    Sev3\n  Labels:      _id=\"20b982ad-cd1d-43f1-b05c-0c221c712ee7\", alertname=\"userJourneyKubeApiserverAvailability1h5m\", cluster=\"ci01-j0627968-mgmt-1\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", name=\"basic-cluster\", namespace=\"ocm-arohcpci01-2s9n9ia6c2epu24n3nrjgvgsrmalvbl6\", resource_id=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-6hk7rf89rgds/providers/microsoft.redhatopenshift/hcpopenshiftclusters/basic-cluster\", severity=\"3\", short_window=\"5m\", slo=\"kas-availability\"\n  Description: Resource ID: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-6hk7rf89rgds/providers/microsoft.redhatopenshift/hcpopenshiftclusters/basic-cluster\nManagement Cluster: ci01-j0627968-mgmt-1\nNamespace: ocm-arohcpci01-2s9n9ia6c2epu24n3nrjgvgsrmalvbl6\n\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:15:57Z\n  Severity:    Sev3\n  Labels:      _id=\"bea7e516-9fcf-42c8-a569-e7972eace539\", alertname=\"userJourneyKubeApiserverAvailability1h5m\", cluster=\"ci01-j0627968-mgmt-2\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", name=\"d3y0h1r9t4i2z7p\", namespace=\"ocm-arohcpci01-2s9n9lf1s41vdkpkfduekbo1vtbqqdr7\", resource_id=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-zstream-upgrade-4-22-8bz42c64rxjn/providers/microsoft.redhatopenshift/hcpopenshiftclusters/cluster-zstream-4-22-n2r9l8\", severity=\"3\", short_window=\"5m\", slo=\"kas-availability\"\n  Description: Resource ID: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-zstream-upgrade-4-22-8bz42c64rxjn/providers/microsoft.redhatopenshift/hcpopenshiftclusters/cluster-zstream-4-22-n2r9l8\nManagement Cluster: ci01-j0627968-mgmt-2\nNamespace: ocm-arohcpci01-2s9n9lf1s41vdkpkfduekbo1vtbqqdr7\n\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:15:57Z\n  Severity:    Sev3\n  Labels:      _id=\"d7014b11-f60b-4daf-aaf4-d29bfef8600a\", alertname=\"userJourneyKubeApiserverAvailability1h5m\", cluster=\"ci01-j0627968-mgmt-2\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", name=\"y7z1l5m6q1x6c2h\", namespace=\"ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8\", resource_id=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-sn968dwgx24q/providers/microsoft.redhatopenshift/hcpopenshiftclusters/basic-hcp-cluster\", severity=\"3\", short_window=\"5m\", slo=\"kas-availability\"\n  Description: Resource ID: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-sn968dwgx24q/providers/microsoft.redhatopenshift/hcpopenshiftclusters/basic-hcp-cluster\nManagement Cluster: ci01-j0627968-mgmt-2\nNamespace: ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8\n\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:15:57Z\n  Severity:    Sev3\n  Labels:      _id=\"2fca8da9-1e0a-4528-82bc-24a68640f216\", alertname=\"userJourneyKubeApiserverAvailability1h5m\", cluster=\"ci01-j0627968-mgmt-1\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", name=\"d7l9h4i9a8u5k9s\", namespace=\"ocm-arohcpci01-2s9n9l6npk7dv191ctaf3b1jnodinlcc\", resource_id=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-serialconsole-bootdiag-q8bm22z27sjd/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster-bootdiag\", severity=\"3\", short_window=\"5m\", slo=\"kas-availability\"\n  Description: Resource ID: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-serialconsole-bootdiag-q8bm22z27sjd/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster-bootdiag\nManagement Cluster: ci01-j0627968-mgmt-1\nNamespace: ocm-arohcpci01-2s9n9l6npk7dv191ctaf3b1jnodinlcc\n\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:16:57Z\n  Severity:    Sev3\n  Labels:      _id=\"e3abb0c7-5f3e-45fa-a6c2-3d0e7a13cc5b\", alertname=\"userJourneyKubeApiserverAvailability1h5m\", cluster=\"ci01-j0627968-mgmt-2\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", name=\"o9i8l3k0v2i2t4i\", namespace=\"ocm-arohcpci01-2s9n9jdesahjtqdjvpc60o8amh8beqdt\", resource_id=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/disabled-image-registry-fbtznfmgpfmh/providers/microsoft.redhatopenshift/hcpopenshiftclusters/disabled-image-registry-hcp-cluster\", severity=\"3\", short_window=\"5m\", slo=\"kas-availability\"\n  Description: Resource ID: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/disabled-image-registry-fbtznfmgpfmh/providers/microsoft.redhatopenshift/hcpopenshiftclusters/disabled-image-registry-hcp-cluster\nManagement Cluster: ci01-j0627968-mgmt-2\nNamespace: ocm-arohcpci01-2s9n9jdesahjtqdjvpc60o8amh8beqdt"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"3a2d3dcfe06b6e61a6b799369fb7b2be07c631aba67738face54f5a991bc059b","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0c158f23d05bb2bcb4ce53b2e8ea76abda6e655547218e7a0fe940e3a9592374","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j2704896 fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j2704896 fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j2704896 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T04:29:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j2704896 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-21T05:47:43Z\n  Ended:       2026-08-21T06:00:37Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 10-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"c8a66989bf0343b565fa3da314117aa183d7f0a34150a848cecfbcf5f963353a","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"827b247c67e0c1c1ff223d0571ae6acbe84e136f93ebb1c1235ad187db53fdf1","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j3334144 fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j3334144 fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j3334144 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T18:10:59Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j3334144 does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-20T19:14:04Z\n  Ended:       2026-08-20T19:16:55Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 10-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-20T19:19:35Z\n  Ended:       2026-08-20T19:38:19Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 10-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090501897923334144","occurred_at":"2026-08-20T18:10:59Z","signature_id":"3c20c1a005d26cb7f8535a4b8df78c801d2bd82791d2fe759399a81b073277dc","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6c86829d1a8522011a815ca18b30a4e79db08ad0649db90f205c775879437d85","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7204608 fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7204608 fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7204608 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T20:15:14Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7204608 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-20T21:11:04Z\n  Ended:       2026-08-20T21:25:19Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 10-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090533136747204608","occurred_at":"2026-08-20T20:15:14Z","signature_id":"c8a66989bf0343b565fa3da314117aa183d7f0a34150a848cecfbcf5f963353a","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"71141656790e0301084c3e739bec3bf4dcc7697951964de2fcf8b30c55b1c3d1","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7691136 fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7691136 fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7691136 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T22:09:10Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7691136 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-20T23:30:29Z\n  Ended:       2026-08-20T23:50:30Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 10-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090566949577691136","occurred_at":"2026-08-20T22:09:10Z","signature_id":"c8a66989bf0343b565fa3da314117aa183d7f0a34150a848cecfbcf5f963353a","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"70db2d61c2cd057fcd1ea1e913f1f94a57b2c2f5dbd99a0ba89a7c674d643f54","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j8947456 fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j8947456 fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j8947456 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j8947456 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T15:54:58Z\n  Ended:       2026-08-25T16:00:59Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 10-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261054988947456","occurred_at":"2026-08-25T14:41:22Z","signature_id":"c8a66989bf0343b565fa3da314117aa183d7f0a34150a848cecfbcf5f963353a","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2cf0526f26a751e656747eaaa8e7450cbcff553172ce32927513b015717916cb","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j9138688 fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j9138688 fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j9138688 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T09:00:23Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j9138688 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-21T10:06:46Z\n  Ended:       2026-08-21T10:20:50Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 10-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090725692609138688","occurred_at":"2026-08-21T09:00:23Z","signature_id":"c8a66989bf0343b565fa3da314117aa183d7f0a34150a848cecfbcf5f963353a","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d281f42786cb2a2de5619c7f3d94cdb69aa344781a27c06b8ed771f436cd9a9a","failure_pattern":"alert [infra] Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j6685312 fired","search_query":"alert [infra] Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j6685312 fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j6685312 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T20:53:01Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j6685312 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-19T21:58:37Z\n  Ended:       2026-08-19T22:05:25Z\n  Severity:    Sev3\n  Description: Cosmos DB returned more than 100 throttled requests (HTTP 429 - request rate too large) summed over a 5-minute window, evaluated every minute. Sustained 429s mean the workload is exceeding provisioned throughput - investigate hot partitions or increase provisioned throughput (RU/s). A low, steady rate of 429s that client SDKs retry transparently can be normal. https://learn.microsoft.com/azure/cosmos-db/troubleshoot-request-rate-too-large"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090180258316685312","occurred_at":"2026-08-19T20:53:01Z","signature_id":"0766b6645c6206a3fc1a103b5ed7d9fe07fd1625b37249763edfe4af9b817fbb","pr_number":6614,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0e72a5ca42c3a2d32bb83d02da4302579e640eee948d64f58a4f543c892be49b","failure_pattern":"alert [infra] Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j7405952 fired","search_query":"alert [infra] Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j7405952 fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j7405952 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j7405952 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T15:54:11Z\n  Ended:       2026-08-25T16:01:15Z\n  Severity:    Sev3\n  Description: Cosmos DB returned more than 100 throttled requests (HTTP 429 - request rate too large) summed over a 5-minute window, evaluated every minute. Sustained 429s mean the workload is exceeding provisioned throughput - investigate hot partitions or increase provisioned throughput (RU/s). A low, steady rate of 429s that client SDKs retry transparently can be normal. https://learn.microsoft.com/azure/cosmos-db/troubleshoot-request-rate-too-large"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092261270257405952","occurred_at":"2026-08-25T14:42:10Z","signature_id":"0766b6645c6206a3fc1a103b5ed7d9fe07fd1625b37249763edfe4af9b817fbb","pr_number":6614,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T16:39:13Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T17:24:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j9953536-svc\", component=\"monitoring-infrastructure\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j9953536-svc has been unreachable for 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090116384229953536","occurred_at":"2026-08-19T16:39:13Z","signature_id":"8f47276792f56fd80d601111472f61f301ea1c8057abf8c04c2a4640a8c202fd","pr_number":6601,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c4e2bd93d0410f48ddfa390cc77604492607a58b2a4eaff204baa6c7bed3af07","failure_pattern":"alert [svc] KubeContainerOOMKilled fired","search_query":"alert [svc] KubeContainerOOMKilled fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[3,0,0,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-18T19:17:13Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-24T16:04:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-cxwt8 on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-24T16:04:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-jbtcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1ec91183-7c07-473b-b1d1-9df325a61c81\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-jbtcb on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-24T16:04:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-jbtcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1ec91183-7c07-473b-b1d1-9df325a61c81\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-jbtcb on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-24T16:04:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-cxwt8 on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"c239e70f5bf7ad0e42856cb2b9acc6d7c10a9ba7d6c9a4f5496cbd3609446018","pr_number":5958,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":15,"prior_last_seen_at":"2026-08-21T01:50:52Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T03:16:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6439552-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6439552/providers/microsoft.monitor/accounts/services-j6439552\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-6t89j\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T03:16:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6439552-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6439552/providers/microsoft.monitor/accounts/services-j6439552\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-6t89j\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"1532964a3b40e3b2ec3338f5c64978e5eabba2056b68a3c8f4a6cbc6a64ca184","pr_number":6617,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-13T06:28:29Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T16:19:07Z\n  Ended:       2026-08-24T16:30:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n  Description: Pod arobit/arobit-forwarder-cxwt8 (fluentbit) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T16:20:06Z\n  Ended:       2026-08-24T16:33:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n  Description: Pod arobit/arobit-forwarder-cxwt8 (fluentbit) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"fe91f4f69f02671271e4cc6e4aaa4f966388d16d9e660f0bf8e6f1f0d5179d16","pr_number":5958,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fb0cbad5198bdae4515e07599d2b1b6ff3ac6cb7da7109069ce8cd804ef1fb4b","failure_pattern":"alert [svc] PrometheusJobUp fired","search_query":"alert [svc] PrometheusJobUp fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert PrometheusJobUp does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-19T16:39:13Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert PrometheusJobUp does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T17:14:16Z\n  Severity:    Sev2\n  Labels:      alertname=\"PrometheusJobUp\", cluster=\"ci01-j9953536-svc\", component=\"monitoring-infrastructure\", severity=\"critical\"\n  Description: Prometheus has not been reachable for the past 10 minutes.\nThis may indicate that the Prometheus server is down, unreachable due to network issues, or experiencing a crash loop.\nCheck the status of the Prometheus pods, service endpoints, and network connectivity."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090116384229953536","occurred_at":"2026-08-19T16:39:13Z","signature_id":"8622c4f05e530429c2435ac6bd159aff27f3109548086ec61b7cd9ecf8f87649","pr_number":6601,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"e650175e12814e322599e325ba41102fb2cdbd85011eb93727df4167243d0b83","failure_pattern":"all 1 initial nodes should be Ready","search_query":"all 1 initial nodes should be Ready","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T11:53:35Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:174]: Timed out after 1200.004s.\nall 1 initial nodes should be Ready\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:170 with:\nexpected 1 nodes, got output: \nExpected\n    \u003c[]string | len:0, cap:0\u003e: ...\nto have length 1\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:174]: Timed out after 1200.004s.\nall 1 initial nodes should be Ready\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:170 with:\nexpected 1 nodes, got output: \nExpected\n    \u003c[]string | len:0, cap:0\u003e: ...\nto have length 1"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090769277882732544","occurred_at":"2026-08-21T11:53:35Z","signature_id":"7dfbb53772571c55a08382817583ca911847760779d9cb56c53bd66f1d25a026","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"48a64ec1b4b5565d2f7769f9a521ab2dc62d4c19c2f03ba77ff5b2f0a1ca7e6e","failure_pattern":"can't list nodes in the cluster: Get \"\u003curl\u003e","search_query":"can't list nodes in the cluster: Get \"https://10","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-20T17:48:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:153]: failed to verify initial node count of 1\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000bf6980\u003e: \n    can't list nodes in the cluster: Get \"https://10.0.0.5:443/api/v1/nodes\": dial tcp 10.0.0.5:443: i/o timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:153]: failed to verify initial node count of 1\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000bf6980\u003e: \n    can't list nodes in the cluster: Get \"https://10.0.0.5:443/api/v1/nodes\": dial tcp 10.0.0.5:443: i/o timeout\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"871c1ef42ecb907672fb82bd7b0f6bfc59c5eb5138fb8df2ac817dd02005754a","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5dd5ef6724beffab1f42a094be91cb72cdbac22ffe68c01aeeec9da825c19fa3","failure_pattern":"error for empty VM name should contain 400 status code","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:359]: error for empty VM name should contain 400 status code","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:359]: error for empty VM name should contain 400 status code\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:359]: error for empty VM name should contain 400 status code\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092242102967603200","occurred_at":"2026-08-25T13:26:03Z","signature_id":"2f2e10c3f96af5d17d27eaefc2cdd8f45ad43cf97747092ff970d29300a33cb8","pr_number":6660,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9a4507f47f0192d158049af8a07afbd1c7ecccdd86ac7e743f024025d7a1dd1b","failure_pattern":"failed to create HCP cluster \u003ccluster\u003e: failed starting cluster creation \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\": Get \"\u003curl\u003e\": stream error: stream ID 7; CANCEL; received from peer","search_query":"failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-tslqh4hrqks5\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-tslqh4hrqks5?api-version=2021-04-01\": stream error: stream ID 7; CANCEL; received from peer","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T02:50:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00111c6c0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-tslqh4hrqks5\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-tslqh4hrqks5?api-version=2021-04-01\": stream error: stream ID 7; CANCEL; received from peer\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00111c6c0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-tslqh4hrqks5\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-tslqh4hrqks5?api-version=2021-04-01\": stream error: stream ID 7; CANCEL; received from peer\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089907805891858432","occurred_at":"2026-08-19T02:50:24Z","signature_id":"f81f115cfe4ce2fb632cb9ef6f763ee8ec7e8f43bfe7db385ffdc9661d773ea9","pr_number":6583,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"51411f97b0ca6a75b45db7faf3ee1eadf268adaa51892a177659837a8dfa1072","failure_pattern":"failed to locate cilium chart: looks like \"\u003curl\u003e\" is not a valid chart repository or cannot be reached: Get \"\u003curl\u003e\": dial tcp \u003cip\u003e: connect: network is unreachable","search_query":"failed to locate cilium chart: looks like \"https://helm.cilium.io/\" is not a valid chart repository or cannot be reached: Get \"https://helm.cilium.io/index.yaml\": dial tcp [2606:50c0:8003::153]:443: connect: network is unreachable","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-22T18:27:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:310]: failed to install Cilium chart via Helm SDK\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007d28e0\u003e: \n    failed to locate cilium chart: looks like \"https://helm.cilium.io/\" is not a valid chart repository or cannot be reached: Get \"https://helm.cilium.io/index.yaml\": dial tcp [2606:50c0:8003::153]:443: connect: network is unreachable\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:310]: failed to install Cilium chart via Helm SDK\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007d28e0\u003e: \n    failed to locate cilium chart: looks like \"https://helm.cilium.io/\" is not a valid chart repository or cannot be reached: Get \"https://helm.cilium.io/index.yaml\": dial tcp [2606:50c0:8003::153]:443: connect: network is unreachable\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091230843602210816","occurred_at":"2026-08-22T18:27:40Z","signature_id":"8dcc88eb87847bea80d8d24ee0235e83a6c1dc7f3e44fd2468c0d050417f6533","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fcdc8973bc72f1f3eccc8e8fa24d75136d05bdbdaec8efce65b8dbababc7317e","failure_pattern":"failed to lookup subscription ID for \"\u003credacted-id\u003e\": subscription with name \"\u003credacted-id\u003e\" not found","search_query":"failed to lookup subscription ID for \\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\": subscription with name \\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\" not found","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/global/svc-acr-replication","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-22T02:32:25Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/global/svc-acr-replication","occurrences":1}],"full_error_samples":["time=2026-08-22T02:55:00.244Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=global step=svc-acr-replication err=\"failed to lookup subscription ID for \\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\": subscription with name \\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\" not found\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090990445487198208","occurred_at":"2026-08-22T02:32:25Z","signature_id":"4370e90013292c27b98a5f1edf1b48ee6743a527ca936997a98da21b3f0b0d01","pr_number":6607,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"95452bdc0ded378689e5f2c12b2d45cbf916f7a4e0676d618a92d0f3d8778ac4","failure_pattern":"failed to read rendered config /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: open /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: no such file or directory","search_query":"failed to read rendered config /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: open /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: no such file or directory","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:95]: failed to load rendered config — is RENDERED_CONFIG set?\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eafac0\u003e: \n    failed to read rendered config /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: open /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: no such file or directory\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:95]: failed to load rendered config — is RENDERED_CONFIG set?\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eafac0\u003e: \n    failed to read rendered config /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: open /opt/app-root/src/github.com/Azure/ARO-HCP/config/rendered/dev/ci01/canadacentral.yaml: no such file or directory\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092194838442151936","occurred_at":"2026-08-25T10:18:15Z","signature_id":"18cefbe16bc079f2ead9cb47f111db79d94e08b882e7b91731e46458212a977b","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0c8f285ebf7a77151b5c5324a77de4ff3bf14831af7586b962d27c5eacb2a8e6","failure_pattern":"failed to roll out Helm release: resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Agent/management/deploy (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[1,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-21T20:12:06Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Agent/management/deploy (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-08-21T20:50:16.519Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-08-21T20:50:17.093Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 values=\"map[azure:map[clientId:e8f047ac-b81e-4523-abc9-39d8133a1340 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] broker:map[host:arohcp-ci01-maestro-j3460096.westus3-1.ts.eventgrid.azure.net port:8883] consumerName:hcp-underlay-j3460096-mgmt-2 credsKeyVault:map[name:ah-ci01-mg-j3460096-2 secret:hcp-underlay-j3460096-mgmt-2] deployment:map[zoneCount:4] glog_v:2 image:map[digest:sha256:46e44a9a9cfbd4e157110a6dcc88abdfd569b49315b4ad5088eb42fb69b45b60 registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] installAppliedManifestWorkCRD:false replicas:1 serviceAccountName:maestro sideCar:map[image:map[digest:sha256:bf23c3b25d0f0383736ab2380ed90a25cf5d7aae416de00accc27f8ed6c2e42c registry:mcr.microsoft.com repository:azurelinux/base/nginx]]]\"\ntime=2026-08-21T20:50:17.093Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2\ntime=2026-08-21T20:50:17.310Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 namespace=maestro\ntime=2026-08-21T20:50:17.570Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-08-21T20:50:17.570Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2\ntime=2026-08-21T20:50:17.785Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2\ntime=2026-08-21T20:55:24.253Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 err=\"resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-08-21T20:55:24.253Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894760083460096","occurred_at":"2026-08-21T20:12:06Z","signature_id":"17bb70335ce31ff41cd691af5d92dd814bd463359d91b7f6775654b4403502e7","pr_number":6631,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"2ec88ca94a38e80e8fd52c34dd8310e94dbbbdba81fa0e5a803fb5edd3ec27ed","failure_pattern":"priv-np-scale-np-scale-5kns9-rhgcf Ready worker 64m v1.33.13","search_query":"priv-np-scale-np-scale-5kns9-rhgcf Ready worker 64m v1.33.13","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-23T03:43:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:270]: Timed out after 3600.003s.\nall 1 nodes should be Ready after scale down\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:265 with:\nexpected 1 nodes after scale down, got output: priv-np-scale-np-scale-5kns9-7989x   Ready   worker   73m   v1.33.13\npriv-np-scale-np-scale-5kns9-rhgcf   Ready   worker   64m   v1.33.13\nExpected\n    \u003c[]string | len:2, cap:2\u003e: ...\nto have length 1\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:270]: Timed out after 3600.003s.\nall 1 nodes should be Ready after scale down\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:265 with:\nexpected 1 nodes after scale down, got output: priv-np-scale-np-scale-5kns9-7989x   Ready   worker   73m   v1.33.13\npriv-np-scale-np-scale-5kns9-rhgcf   Ready   worker   64m   v1.33.13\nExpected\n    \u003c[]string | len:2, cap:2\u003e: ...\nto have length 1"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091370674797940736","occurred_at":"2026-08-23T03:43:19Z","signature_id":"e7c295a04eccfabf8a50b991c26c147dc75c254f70efb08c9abc192453da473a","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e131b6de8aab0d4d1241bdfbe5183bba73583ecd4d404f6e1056b123e86af48c","failure_pattern":"priv-np-scale-np-scale-mjh2n-txqzj Ready worker 22m v1.33.13","search_query":"priv-np-scale-np-scale-mjh2n-txqzj Ready worker 22m v1.33.13","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-22T18:27:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:267]: Timed out after 1200.003s.\nall 1 nodes should be Ready after scale down\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:263 with:\nexpected 1 nodes after scale down, got output: priv-np-scale-np-scale-mjh2n-k28ns   Ready   worker   30m   v1.33.13\npriv-np-scale-np-scale-mjh2n-txqzj   Ready   worker   22m   v1.33.13\nExpected\n    \u003c[]string | len:2, cap:2\u003e: ...\nto have length 1\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:267]: Timed out after 1200.003s.\nall 1 nodes should be Ready after scale down\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:263 with:\nexpected 1 nodes after scale down, got output: priv-np-scale-np-scale-mjh2n-k28ns   Ready   worker   30m   v1.33.13\npriv-np-scale-np-scale-mjh2n-txqzj   Ready   worker   22m   v1.33.13\nExpected\n    \u003c[]string | len:2, cap:2\u003e: ...\nto have length 1"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091230843602210816","occurred_at":"2026-08-22T18:27:40Z","signature_id":"9bf5d57efcb7e35f2e7257cf5c373a661b2c0673357fc56c9a7f7dfa748879da","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d47ff99fd1bcd56f3dd1a2d45fb345dc45ebaade86da8c2155a6cfa662ad0280","failure_pattern":"public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud on 172.30.0.10:53: no such host","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T14:35:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:190]: Timed out after 600.002s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:188 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud on 172.30.0.10:53: no such host\nUnexpected error:\n    \u003c*url.Error | 0xc002a34a20\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:190]: Timed out after 600.002s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:188 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud on 172.30.0.10:53: no such host\nUnexpected error:\n    \u003c*url.Error | 0xc002a34a20\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud on 172.30.0.10:53: no such host\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085264507539456","occurred_at":"2026-08-19T14:35:33Z","signature_id":"d14499ebb4d5737a04844d1f8159573bec53cf37189db5b4f855c279fdd04f6b","pr_number":6607,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5d433008aba23c344781b86d474b8dbf122c3c1ce0be7c5095e73ef5b595140f","failure_pattern":"run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"\u003cvm\u003e\" (output: \"Client Version: v1.36.4 Kustomize Version: v5.8.1\")","search_query":"run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"private-kas-test-vm\" (output: \"Client Version: v1.36.4\\nKustomize Version: v5.8.1\")","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.15772870662460567,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:160]: kubectl version should succeed from VM via private KAS internal LB (output: )\nUnexpected error:\n    \u003c*errors.errorString | 0xc0007fbf60\u003e: \n    run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"private-kas-test-vm\" (output: \"Client Version: v1.36.4\\nKustomize Version: v5.8.1\")\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:160]: kubectl version should succeed from VM via private KAS internal LB (output: )\nUnexpected error:\n    \u003c*errors.errorString | 0xc0007fbf60\u003e: \n    run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"private-kas-test-vm\" (output: \"Client Version: v1.36.4\\nKustomize Version: v5.8.1\")\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879954055696384","occurred_at":"2026-08-24T13:27:00Z","signature_id":"d079461036286dee8f18df87fbaabb5b38f1fcaa64aea1bbd3a2fe3d558a9ddb","pr_number":6429,"after_last_push_of_merged_pr":false}]}]},{"environment":"int","summary":{"total_runs":26,"failed_runs":10,"raw_occurrences":12,"matched_occurrences":10,"runs_affected":8},"rows":[{"environment":"int","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.6923076923076925,"anchor_occurrences":2,"after_last_push_count":7,"also_in":["PROD","STG"],"trend_counts":[1,0,1,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-20T16:42:49Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-l8xph5\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f20140\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-trf9f.apps.aro.r8g4l2d8f3g0v2i.b5du.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-l8xph5\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f20140\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-trf9f.apps.aro.r8g4l2d8f3g0v2i.b5du.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000c8e2d0\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-w89wr.apps.aro.cilium-cl.ecqe.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000c8e2d0\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-w89wr.apps.aro.cilium-cl.ecqe.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091735435544891392","occurred_at":"2026-08-24T03:52:45Z","signature_id":"571e7c77690730a14fa42704d42877387a3fe5f21673f5f238b11729008a3c86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2090479680229478400","occurred_at":"2026-08-20T16:42:49Z","signature_id":"dea774d2681150a0cd95cf0e47e56b2fea06f005627625b51ed01c7c9d66a8f9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"f36a1bbc5e9fcc844fe3fa2cebef1b990c1c4028fdbe37fca0988cbc08f26ddd","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; detail message Another operation is pending for requested object. Operation group '/operations/groups/id/|subscriptions|XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX|resourceGroups|private-keyvault-459rvjqk6tld|privateDnsZones|privatelink.vaultcore.azure.net' already has 1 operations like '/operations/type/UpsertPrivateDnsZone/id/822c0f2c-dbd9-4336-bdd7-54dca0cd3e73_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' queued.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T08:43:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:75]: failed to create customer resources with private key vault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dee6c0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-private-kv-cluster-q9q757\" in resourcegroup=\"private-keyvault-459rvjqk6tld\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-keyvault-459rvjqk6tld/providers/Microsoft.Resources/deployments/customer-infra-private-kv-cluster-q9q757/operationStatuses/08584143040338090197\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"code\\\": \\\"Conflict\\\",\\r\\n  \\\"message\\\": \\\"Another operation is pending for requested object. Operation group '/operations/groups/id/|subscriptions|XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX|resourceGroups|private-keyvault-459rvjqk6tld|privateDnsZones|privatelink.vaultcore.azure.net' already has 1 operations like '/operations/type/UpsertPrivateDnsZone/id/822c0f2c-dbd9-4336-bdd7-54dca0cd3e73_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' queued.\\\"\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:75]: failed to create customer resources with private key vault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dee6c0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-private-kv-cluster-q9q757\" in resourcegroup=\"private-keyvault-459rvjqk6tld\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-keyvault-459rvjqk6tld/providers/Microsoft.Resources/deployments/customer-infra-private-kv-cluster-q9q757/operationStatuses/08584143040338090197\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"code\\\": \\\"Conflict\\\",\\r\\n  \\\"message\\\": \\\"Another operation is pending for requested object. Operation group '/operations/groups/id/|subscriptions|XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX|resourceGroups|private-keyvault-459rvjqk6tld|privateDnsZones|privatelink.vaultcore.azure.net' already has 1 operations like '/operations/type/UpsertPrivateDnsZone/id/822c0f2c-dbd9-4336-bdd7-54dca0cd3e73_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' queued.\\\"\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2090721494970667008","occurred_at":"2026-08-21T08:43:42Z","signature_id":"0760817b38be15b0376ec42c964f4fcf6c68007f8f83cfdd5501d0d518b4010f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"19c7731fda7f45e3249a335bc7ba4753eeec988f740bb716b85a25c2a5edd90f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD","STG"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T05:35:18Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000729200\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-9m9kd7rcmdds\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83f17bd3-dd33-461c-a6bb-3c1efcddd145\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83f17bd3-dd33-461c-a6bb-3c1efcddd145\",\n      \"name\": \"83f17bd3-dd33-461c-a6bb-3c1efcddd145\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T06:19:56.001094985Z\",\n      \"endTime\": \"2026-08-19T06:38:56.416812667Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000729200\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-9m9kd7rcmdds\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83f17bd3-dd33-461c-a6bb-3c1efcddd145\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83f17bd3-dd33-461c-a6bb-3c1efcddd145\",\n      \"name\": \"83f17bd3-dd33-461c-a6bb-3c1efcddd145\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T06:19:56.001094985Z\",\n      \"endTime\": \"2026-08-19T06:38:56.416812667Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089949305547788288","occurred_at":"2026-08-19T05:35:18Z","signature_id":"fdf3255b7b121e3a193ebc95f7ddc22d058f11ef8f585db76055ebe3aa3840be","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"13ca13618a2c4b8b7429b3001f7d02e419916c0961928bfe37af66198711655d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":7,"also_in":["DEV","PROD","STG"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-03"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-07T23:58:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e43900\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fbxfthcgbs8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n      \"name\": \"7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T10:43:39.73808157Z\",\n      \"endTime\": \"2026-08-24T11:02:45.476245961Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e43900\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fbxfthcgbs8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n      \"name\": \"7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T10:43:39.73808157Z\",\n      \"endTime\": \"2026-08-24T11:02:45.476245961Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091834116805758976","occurred_at":"2026-08-24T10:24:52Z","signature_id":"3fa15812aed9d226c2b6bdead73b3a2c80d16497f78ee163ab8cbfa1074904ed","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0e49f848ca7d69ae73432116f8c3e2d9ecf6b571282ffce3e6703b901225aa88","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD","STG"],"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-23T22:07:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-kjzq8f with version 4.21.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004b0bc0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-kjzq8f: failed waiting for cluster=\"np-version-upgrade-cluster-kjzq8f\" in resourcegroup=\"rg-np-version-upgrade-kjzq8f-kc56h5fmvk6m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8dadd8ad-eeaa-4dea-bb90-c4f1255a79d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8dadd8ad-eeaa-4dea-bb90-c4f1255a79d0\",\n      \"name\": \"8dadd8ad-eeaa-4dea-bb90-c4f1255a79d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T23:22:28.180803579Z\",\n      \"endTime\": \"2026-08-23T23:41:32.080303176Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-kjzq8f with version 4.21.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004b0bc0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-kjzq8f: failed waiting for cluster=\"np-version-upgrade-cluster-kjzq8f\" in resourcegroup=\"rg-np-version-upgrade-kjzq8f-kc56h5fmvk6m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8dadd8ad-eeaa-4dea-bb90-c4f1255a79d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8dadd8ad-eeaa-4dea-bb90-c4f1255a79d0\",\n      \"name\": \"8dadd8ad-eeaa-4dea-bb90-c4f1255a79d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T23:22:28.180803579Z\",\n      \"endTime\": \"2026-08-23T23:41:32.080303176Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091648540576059392","occurred_at":"2026-08-23T22:07:27Z","signature_id":"0fa8c01ab0585932acfc8df37080798105e378ae78d09a1e92de90a78ac7d573","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"86acf58965dfb7b96713f6a89ff381ba46ffdd44516e1b61638ec23fb893f80b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":6,"also_in":["DEV","PROD","STG"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-19T19:42:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000c26120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-n4cgqv\" in resourcegroup=\"rg-np-version-upgrade-n4cgqv-22qb5ssqb72c\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0f6000cd-5626-4ea1-84d3-e3c4775f5ae6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0f6000cd-5626-4ea1-84d3-e3c4775f5ae6\",\n      \"name\": \"0f6000cd-5626-4ea1-84d3-e3c4775f5ae6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T21:24:56.829945477Z\",\n      \"endTime\": \"2026-08-19T21:48:59.897240723Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9shr1s413r5943057b8kmkq36ndccb still exists (deletion dispatched at 2026-08-19T21:24:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000c26120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-n4cgqv\" in resourcegroup=\"rg-np-version-upgrade-n4cgqv-22qb5ssqb72c\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0f6000cd-5626-4ea1-84d3-e3c4775f5ae6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0f6000cd-5626-4ea1-84d3-e3c4775f5ae6\",\n      \"name\": \"0f6000cd-5626-4ea1-84d3-e3c4775f5ae6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T21:24:56.829945477Z\",\n      \"endTime\": \"2026-08-19T21:48:59.897240723Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9shr1s413r5943057b8kmkq36ndccb still exists (deletion dispatched at 2026-08-19T21:24:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2090162390078328832","occurred_at":"2026-08-19T19:42:01Z","signature_id":"83ed69b86924c06c334d849269e853976f842aa7586fae9047aa65459f2dd885","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"69e3ad9bd0c4124153f6e74cb4797f8d44a9ae591984376d6e63dbb2a5d2ca33","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": no upgrade path to update channel \"\u003cocp-channel\u003e\" is currently available for this cluster; a channel appears in the cluster's desired version channels only when an upgrade edge to a release in that channel exists; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T03:16:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:174]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-dxvrk4\" to 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000b99590\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-kbc8vjkst6lr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-dxvrk4\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": no upgrade path to update channel \\\"candidate-4.21\\\" is currently available for this cluster; a channel appears in the cluster's desired version channels only when an upgrade edge to a release in that channel exists\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:174]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-dxvrk4\" to 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000b99590\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-kbc8vjkst6lr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-dxvrk4\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": no upgrade path to update channel \\\"candidate-4.21\\\" is currently available for this cluster; a channel appears in the cluster's desired version channels only when an upgrade edge to a release in that channel exists\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2090639026095656960","occurred_at":"2026-08-21T03:16:00Z","signature_id":"1742306de2425841d8b2f90fcaf3ed04ecc8996c35a137df26e79d915760f264","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"6ae8052fe24068dd122c1d2c69a1fea3114083e0220320db69d2f9ee76d7c61d","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD","STG"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T08:43:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000693590\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \"4.21\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000693590\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \"4.21\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2090721494970667008","occurred_at":"2026-08-21T08:43:42Z","signature_id":"f3bc24e0d223a99b76059ba01e7fa681c39f26ab9d0f4665bfdfdd33c395a153","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"d414eaa781a1a73b840b0ceb0c494d2cfb36d6ce2f01987a892eec84026f3876","failure_pattern":"failed to select node pool \u003cnodepool\u003e version for \u003cocp-channel\u003e channel: Get \"\u003curl\u003e\": net/http: TLS handsh","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": net/http: TLS handsh","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": net/http: TLS handshake timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": net/http: TLS handshake timeout"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091834116805758976","occurred_at":"2026-08-24T10:24:52Z","signature_id":"c4114fce72966a1339b4e9230a4f088e9e9d8e5a8939f917174c276b7d0a2542","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"prod","summary":{"total_runs":35,"failed_runs":23,"raw_occurrences":173,"matched_occurrences":172,"runs_affected":22},"rows":[{"environment":"prod","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":40,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":40,"after_last_push_count":110,"trend_counts":[8,40,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-19T08:22:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:182]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-wh4d6d\" with install version 4.20.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f13300\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-wh4d6d: failed waiting for cluster=\"cp-ystream-upgrade-4-21-wh4d6d\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-jxk96xvwxvvc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15ccb98b-5e2b-47b1-86e9-30fb0729f99b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15ccb98b-5e2b-47b1-86e9-30fb0729f99b\",\n      \"name\": \"15ccb98b-5e2b-47b1-86e9-30fb0729f99b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T09:16:07.820643421Z\",\n      \"endTime\": \"2026-08-19T09:35:08.723020497Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:182]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-wh4d6d\" with install version 4.20.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f13300\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-wh4d6d: failed waiting for cluster=\"cp-ystream-upgrade-4-21-wh4d6d\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-jxk96xvwxvvc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15ccb98b-5e2b-47b1-86e9-30fb0729f99b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15ccb98b-5e2b-47b1-86e9-30fb0729f99b\",\n      \"name\": \"15ccb98b-5e2b-47b1-86e9-30fb0729f99b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T09:16:07.820643421Z\",\n      \"endTime\": \"2026-08-19T09:35:08.723020497Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:593]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f084a0\u003e: \n    failed to create HCP cluster np-downgrade-bb2pdp: failed waiting for cluster=\"np-downgrade-bb2pdp\" in resourcegroup=\"rg-np-downgrade-bb2pdp-78j5hq9mwzxz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b063fb82-4df1-44ba-b542-bd61ed70bc8d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b063fb82-4df1-44ba-b542-bd61ed70bc8d\",\n      \"name\": \"b063fb82-4df1-44ba-b542-bd61ed70bc8d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T09:57:38.906376214Z\",\n      \"endTime\": \"2026-08-19T10:16:40.410448062Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:593]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f084a0\u003e: \n    failed to create HCP cluster np-downgrade-bb2pdp: failed waiting for cluster=\"np-downgrade-bb2pdp\" in resourcegroup=\"rg-np-downgrade-bb2pdp-78j5hq9mwzxz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b063fb82-4df1-44ba-b542-bd61ed70bc8d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b063fb82-4df1-44ba-b542-bd61ed70bc8d\",\n      \"name\": \"b063fb82-4df1-44ba-b542-bd61ed70bc8d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T09:57:38.906376214Z\",\n      \"endTime\": \"2026-08-19T10:16:40.410448062Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-gv9hgw\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d9ea20\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-gv9hgw: failed waiting for cluster=\"cluster-zstream-4-21-gv9hgw\" in resourcegroup=\"rg-zstream-upgrade-4-21-5tt9ll7pp27c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/72a9448e-8164-458c-aad2-98cbc2592898\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/72a9448e-8164-458c-aad2-98cbc2592898\",\n      \"name\": \"72a9448e-8164-458c-aad2-98cbc2592898\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T09:11:52.253118092Z\",\n      \"endTime\": \"2026-08-19T09:30:58.688039675Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-gv9hgw\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d9ea20\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-gv9hgw: failed waiting for cluster=\"cluster-zstream-4-21-gv9hgw\" in resourcegroup=\"rg-zstream-upgrade-4-21-5tt9ll7pp27c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/72a9448e-8164-458c-aad2-98cbc2592898\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/72a9448e-8164-458c-aad2-98cbc2592898\",\n      \"name\": \"72a9448e-8164-458c-aad2-98cbc2592898\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T09:11:52.253118092Z\",\n      \"endTime\": \"2026-08-19T09:30:58.688039675Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"0df77b7d1116b1c04fb388c2383af174d4b6edbaa4d4e5a3744713660e4cd825","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"121ea564f6e09bf6d08a688fb9b47f055a6c5d91e5fd35c38964f9895d0057df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"18c9039bc69cf95c601b8b08c93114e87e827ceec49c97def972c111d9291734","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"24647b076479f96a2634ffafc9740db03c56368574307be170da28d3a936de8f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"251bfa172e1491f88da4c45657ef7300e4fb56b3ae0ea9655004b140adcae167","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"2701a343eb1fd58a3d5797b441c84d009bacbed6965fa9bac1279afda3db8418","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"2ccbe9bb881222347b950b7dd86ea2c3966dbdb29858df2e337e1ca31af5f444","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"2ed0aa0e674602cb7c5b8c192e2ad3415a211fa80d12efb0b021cdf8e984c5cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"308de7cc3c14ec9fc72a6cd8af45a3836968fc6edba5782bc456c6b5a2e9e60a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"374be18ea34adc499786976b50d2dc64d047e18fd5431bb754367b67361b4f99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"377ba9ed8cd7e2433c5048eeb092062b02298eb7c15fbf9bd952f870987cc137","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"3a77481299db92370e1beffa62cb32edd380dac97e190a53fe9fd522442ad3d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"3fd30595a87eff5782ce2c7272e3c28a75152243c69c16e99ffb7477c4fb6e38","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"40a4762f8e74398e439cac9f8cd352ab1bf8be139bb7a59cf0495f2afc520f4c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"4ee15af014c611cc0afbf05913562079c6418194a0746c7629baf22e2cf358f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"55b5d1f08bf9103fe0cfbd2c0bde5e44b3dae7d0845a2c07f727c13c2ed01bb0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"5642d58d8b7f01f3ce33ae5c22193c16d2c94d658b647a580e99eeb646b5e207","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"6f6ba678298e0a25a13f71d351d6e5c0fb1eb55d2124e502903fb9cd124063c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"752382c73c4c0ccf5fe99ae75057e4dc04165a0d62c6bfd96e46376bb189e8ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"774d54b5dde3957244f2491b600d4232c10ba230e6e985998fab3e2de31f0c7a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"7b690261c0fdaf70aefc4e0c1d69295e13b362a2b7899f5851c901bb278dff67","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"80387530c920441616b9e6a753c637d2569df58f0fba8acf80ac68dcac4ddd89","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"90412c7236f5f1b1251aba9404d1630d44bd21548e7ee491d0975760884da1f8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"909047c87a82a5ca0d1c3f8beb70ef026b4c10d4620d27a7fa62d5b013152d2f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"959c55b5753e9b3f9b64e78a7547fcae2646e1a6ade66a7608e02f8e5fd4199c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"99b921f0371a9472fbf21280597188c121a450f3d0a253a45ee3c150a556fca0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"acdd96c0e3a16a7a9f195bb34ead4a7bccfeb4f1e28f74ff353e6516e5a8db21","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"b020a66f78a03802d4210f860b61d90b466c1ebab9b9330375959b3ea3b125df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"b3411c91676226ed4ddc0cba69349300903c5770fda4254a458cc547822b15b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"cb286eb28131d69c054d95d7b3d4393deb3cdba7ec6f2bdce8f6a9fc662ebbe6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"d47910ed53117197469d11f4e78d7397bece9ab04a780d95efb0d35c0c3332c6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"d500f7e2ec150ee3b234a12a903c5d7f46b52dbef3a1576f0d1c22c8e00ad4b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"d8c637988c494a1d61e14bb1a3ffac6a1036745d05dbbaaa70f0b2eded47b391","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"df17467cb7388f00b42f99219d9938f3a5c47ecd51c8e4a52183038b83c64ccd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"df494084b3e8e6856a3071f0cf84ca0d2623b0cfcf0648837f004cfcb22ea6d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"e40cadc23b17461d009accc7c7c5d344bfe4ed6f56fa055f30ddf18580824e71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"ea6d83f9b96725031baa2eaac74488edfa9eb64e7b13aef9ac312df88f744d11","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"f728b1a15e85fd87d77c58b75efc540f29883c7cc8f3857ee12e8efc0f5d1517","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"f92558b5df4deb14668fd647dcfd93e77a702ffb1646fa364db2954ad3f9b2ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"fb7b2959617f8b36a74b71f6f4b06d420ea039ffc68fad695e362f09c04be255","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-mh5jtc, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-mh5jtc in resource group rg-candidate-4-22-7z5ghb47hkdz, error: failed waiting for cluster=\"cluster-candidate-4-22-mh5jtc\" in resourcegroup=\"rg-candidate-4-22-7z5ghb47hkdz\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-mh5jtc in resource group rg-candidate-4-22-7z5ghb47hkdz, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":23,"runs_affected":14,"failed_runs":14,"run_impact_percent":40,"anchor_occurrences":23,"after_last_push_count":23,"also_in":["DEV","STG"],"trend_counts":[0,10,3,2,2,2,3,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":33,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":6}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-zmwf7l76v4hv/cluster-candidate-5-0-p2dtvg should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a42420\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-p2dtvg, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-p2dtvg in resource group rg-candidate-5-0-zmwf7l76v4hv, error: failed waiting for cluster=\"cluster-candidate-5-0-p2dtvg\" in resourcegroup=\"rg-candidate-5-0-zmwf7l76v4hv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-p2dtvg in resource group rg-candidate-5-0-zmwf7l76v4hv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-zmwf7l76v4hv/cluster-candidate-5-0-p2dtvg should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a42420\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-p2dtvg, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-p2dtvg in resource group rg-candidate-5-0-zmwf7l76v4hv, error: failed waiting for cluster=\"cluster-candidate-5-0-p2dtvg\" in resourcegroup=\"rg-candidate-5-0-zmwf7l76v4hv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-p2dtvg in resource group rg-candidate-5-0-zmwf7l76v4hv, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001f06450\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-m4qs88j7zlw8\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001f06450\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-m4qs88j7zlw8\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-c58j4775b54x/cluster-nightly-5-1-rb2jwr should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001a0d800\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-rb2jwr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: failed waiting for cluster=\"cluster-nightly-5-1-rb2jwr\" in resourcegroup=\"rg-nightly-5-1-c58j4775b54x\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-c58j4775b54x/cluster-nightly-5-1-rb2jwr should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001a0d800\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-rb2jwr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: failed waiting for cluster=\"cluster-nightly-5-1-rb2jwr\" in resourcegroup=\"rg-nightly-5-1-c58j4775b54x\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2092069681539059712","occurred_at":"2026-08-25T02:00:55Z","signature_id":"f812b5873fe0043e9314ca322d6d4251f24251a8ffe19fbea9df860ac64ea470","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"48cf5ec4badd8d65b424a299cc4f30c373af95f5d42c09e6725e0ee15b8c1667","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"a75e2b44cb494e9ca286b15ddee1a744c32c76fd89cb2c425050bd89a5961c7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"5fc7819289b1249d9f2299f61167ea9b6a16c06d2102e06da9f3562ee86b0ad0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091661989947379712","occurred_at":"2026-08-23T23:00:54Z","signature_id":"9c9613ddf68cabbd7bf2597c127a096449a4ab4a704124ea9fdfd629dd68d4d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091344896685248512","occurred_at":"2026-08-23T02:00:53Z","signature_id":"4bff708c71d99dabe7a7ae9698b7c41e55c0d1e72df16dff74af2e7f9509686b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"47702ee742345d64441e042bfdcfa97bab40d3a508721722eba88a7afc082a55","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090982470148165632","occurred_at":"2026-08-22T02:00:44Z","signature_id":"2757e229110f4811519f1ff50a6973559f0450d7596fa852667ff0797de1e8f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","signature_id":"542d678565bcb4bd14e18783f6cd563900f9c155e36aa7035ac1070cce39883f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","signature_id":"0b9c27a850dd92839570250cc1ea4c147b359cb2854e15193a15f04b24871364","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090574634771550208","occurred_at":"2026-08-20T23:00:08Z","signature_id":"41a62bc60eab6b5ddcd3bdfb1285a3b6400326d86961ab39390e2f9f677abe8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090574634771550208","occurred_at":"2026-08-20T23:00:08Z","signature_id":"aacace58e98875c1225bb8a8744e8e8416eb18bc3700b8451e6cabf41bd0ffad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090257540720889856","occurred_at":"2026-08-20T02:00:07Z","signature_id":"d37b69e263fbf930ec5cde42a4763ff2e6dfc059aac58455e214660d9ff281b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090212357799153664","occurred_at":"2026-08-19T23:00:07Z","signature_id":"48db361fd6cbe53d4b314013e1f67f702ce3583723a51cc22976b7761d1e269a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090212357799153664","occurred_at":"2026-08-19T23:00:07Z","signature_id":"4eb1908867ea617b79ef714731840e3f597cbcd49098bec1a8c9d6b2ce2b7421","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"5c48449131a8d336b23406dd697ca42f090d1dc3c91e24c0f6321967915f23f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"7b07eb09a717b0f9075e8c0dbd5167dceb9caf5a8c49f226b9844c0759771559","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"7e524566c996be43d028e5a4f51cdbd9701e5d3b749912cf8b628227b47bc7c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"d9640b3d4577c59f0336a7cadb9c6f63f512e507dba864340fe337ec82787edb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"e362871c873024a97bfec201061c45353b63e2faf5024b5e599e89ca47286de6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"ec12fc460973c9fa872cf3c9849f1f17d27d6d036db485d805694205ed371d6b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"081f1a38953655e4ae6091a8971cca6c1ebceef583b868a8aac6f4e70f09252c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"3747e3fe8cf830b2dbca441528582abc994eebf4e1accb8b03ec49960570f75a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":17,"runs_affected":4,"failed_runs":4,"run_impact_percent":11.428571428571429,"anchor_occurrences":17,"after_last_push_count":91,"also_in":["DEV","STG"],"trend_counts":[36,13,0,0,4,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":29,"prior_last_seen_at":"2026-08-22T23:00:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-21-5t5lsf5rjn4g/cluster-nightly-4-21-xmf9vj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009ad020\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-xmf9vj: failed waiting for cluster=\"cluster-nightly-4-21-xmf9vj\" in resourcegroup=\"rg-nightly-4-21-5t5lsf5rjn4g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f9d217a-1f3c-416d-8de9-636190e4122d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f9d217a-1f3c-416d-8de9-636190e4122d\",\n      \"name\": \"5f9d217a-1f3c-416d-8de9-636190e4122d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:29:39.828760367Z\",\n      \"endTime\": \"2026-08-22T23:48:43.906494075Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-21-5t5lsf5rjn4g/cluster-nightly-4-21-xmf9vj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009ad020\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-xmf9vj: failed waiting for cluster=\"cluster-nightly-4-21-xmf9vj\" in resourcegroup=\"rg-nightly-4-21-5t5lsf5rjn4g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f9d217a-1f3c-416d-8de9-636190e4122d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f9d217a-1f3c-416d-8de9-636190e4122d\",\n      \"name\": \"5f9d217a-1f3c-416d-8de9-636190e4122d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:29:39.828760367Z\",\n      \"endTime\": \"2026-08-22T23:48:43.906494075Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014ce920\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-smph858sdl8w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b54e40a6-a611-4bbe-8757-d1d8cc90bbc9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b54e40a6-a611-4bbe-8757-d1d8cc90bbc9\",\n      \"name\": \"b54e40a6-a611-4bbe-8757-d1d8cc90bbc9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:04:17.915889936Z\",\n      \"endTime\": \"2026-08-22T23:23:21.720195497Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014ce920\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-smph858sdl8w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b54e40a6-a611-4bbe-8757-d1d8cc90bbc9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b54e40a6-a611-4bbe-8757-d1d8cc90bbc9\",\n      \"name\": \"b54e40a6-a611-4bbe-8757-d1d8cc90bbc9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:04:17.915889936Z\",\n      \"endTime\": \"2026-08-22T23:23:21.720195497Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f80e40\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-d2gz4r6tmvpw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c45d03f9-8ce4-4648-8394-8c6d842c5331\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c45d03f9-8ce4-4648-8394-8c6d842c5331\",\n      \"name\": \"c45d03f9-8ce4-4648-8394-8c6d842c5331\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:43:24.031623519Z\",\n      \"endTime\": \"2026-08-23T00:02:33.649737296Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f80e40\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-d2gz4r6tmvpw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c45d03f9-8ce4-4648-8394-8c6d842c5331\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c45d03f9-8ce4-4648-8394-8c6d842c5331\",\n      \"name\": \"c45d03f9-8ce4-4648-8394-8c6d842c5331\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:43:24.031623519Z\",\n      \"endTime\": \"2026-08-23T00:02:33.649737296Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"07048632dc11c59fdf44740c1ef4e0eb9f45fa439ab3b48e7a52ad336d44f81c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"1f772bd9d9a81506f67e6f590fc47b7eb94e1511b429f44411c93cafd6da7abf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"4a1d354c4397ba3bb89eaf00c81e13c1e76974c518f83a48ea93a10d55e1d0fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"860de345a8938b4d31a30e61d384ad788970489b8c1a04f8d2964c010566a745","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089972004693217280","occurred_at":"2026-08-19T07:04:47Z","signature_id":"de68c7f1eed1daba4792282100d6af1320f2ddc0cb0b268d2b656a8f240b65b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089901294310395904","occurred_at":"2026-08-19T02:24:31Z","signature_id":"294303ab3f0803f5e3201beb670ce1fbfdde703c5d9b5762ab488911185921af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089901294310395904","occurred_at":"2026-08-19T02:24:31Z","signature_id":"adad22468ff9be4b330d88b35e123924c23f8c72219649a6db53350d7a9c0969","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"17a5a27eceaf85709d921f2350714ee653956ec3dce32061b6d2f3d140782f28","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"4d6dee469821e05603630a76eb689956f731af3a89f51c6c199dd524171ec578","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"6ecd34deef78788afd184d4cf3964dbb22bf21a40747f667791db8892dff5003","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"8a1aeb73c86777a710aebb6152ebc210f77bc8f4f677ec8fc7969644072061f8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"97ff50bba7aa31c13e4678e9a9632fb3db34bf9078076b8842c458a51f34a45e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"a72d651993eddb1be51bcdf8348aeb0eb0fea781b88df0939ec404816eebfd02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"c1066041e12c007a97ab6f62a064213658033f1574328a8adae079e950f3c0f4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"d2a799c25d7e109674af43387baa7b553d3c8e1554a4dfaf3d6785f5da0811f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"d45996193f19fe48fdbc1ef49a2cefe2375e2a2228418ba3eff3c51e456a8a08","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"e4327a3940ec731c1cf75bb7184697633b96a1e968205743c14d33e37455381a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"97958430e3a844b6f5be05213043088dddda4827bc264aa7b71a4bdf2cce51d5","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":9,"runs_affected":6,"failed_runs":6,"run_impact_percent":17.142857142857142,"anchor_occurrences":9,"after_last_push_count":10,"also_in":["DEV","STG"],"trend_counts":[0,5,0,0,0,1,0,3],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-17"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00093afe0\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-bm5z6ztxsxcf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56042e31-0efc-4287-925a-c64efa21388d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56042e31-0efc-4287-925a-c64efa21388d\",\n      \"name\": \"56042e31-0efc-4287-925a-c64efa21388d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T10:30:08.889821647Z\",\n      \"endTime\": \"2026-08-25T10:49:13.717723647Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00093afe0\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-bm5z6ztxsxcf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56042e31-0efc-4287-925a-c64efa21388d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56042e31-0efc-4287-925a-c64efa21388d\",\n      \"name\": \"56042e31-0efc-4287-925a-c64efa21388d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T10:30:08.889821647Z\",\n      \"endTime\": \"2026-08-25T10:49:13.717723647Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006d1700\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-dnqrhz4h5n9n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/26ec1bc7-0ddd-42ef-916c-12f4d896455c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/26ec1bc7-0ddd-42ef-916c-12f4d896455c\",\n      \"name\": \"26ec1bc7-0ddd-42ef-916c-12f4d896455c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T07:34:58.109914662Z\",\n      \"endTime\": \"2026-08-25T07:53:59.934466955Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006d1700\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-dnqrhz4h5n9n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/26ec1bc7-0ddd-42ef-916c-12f4d896455c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/26ec1bc7-0ddd-42ef-916c-12f4d896455c\",\n      \"name\": \"26ec1bc7-0ddd-42ef-916c-12f4d896455c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T07:34:58.109914662Z\",\n      \"endTime\": \"2026-08-25T07:53:59.934466955Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a308e0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ks6pxxnlhsjm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc3b6654-559a-489d-a192-908217a06b8f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc3b6654-559a-489d-a192-908217a06b8f\",\n      \"name\": \"fc3b6654-559a-489d-a192-908217a06b8f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T07:33:44.141359446Z\",\n      \"endTime\": \"2026-08-25T07:52:49.875150436Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a308e0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ks6pxxnlhsjm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc3b6654-559a-489d-a192-908217a06b8f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc3b6654-559a-489d-a192-908217a06b8f\",\n      \"name\": \"fc3b6654-559a-489d-a192-908217a06b8f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T07:33:44.141359446Z\",\n      \"endTime\": \"2026-08-25T07:52:49.875150436Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092192947897372672","occurred_at":"2026-08-25T10:10:43Z","signature_id":"ad66320c5357dd06c4fac6f8b26348313ed54ba4b4268b4e00a204574fabedc1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092146624879923200","occurred_at":"2026-08-25T07:06:40Z","signature_id":"16e9f20862be2ffa8d9254ea133442622a04c2230e897bc0c633345d949e6e0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2092146624879923200","occurred_at":"2026-08-25T07:06:40Z","signature_id":"837b614da929eed13cefb96defae6f8e2c3ed1ab2105b3725cca3c17f8b63fa0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091344896685248512","occurred_at":"2026-08-23T02:00:53Z","signature_id":"68cb4cfbfaec2dfcb0d787d2c40249532b9f90d9b8bbcba6948bf143a5c89648","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089972004693217280","occurred_at":"2026-08-19T07:04:47Z","signature_id":"8371b215a7c62dde0c1a0f510284a45a0fba9fc6fad21ce69da4e00a46d13002","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089901294310395904","occurred_at":"2026-08-19T02:24:31Z","signature_id":"9287d64e5b4bf5b0b94c17318f08eae05c3e7c9aefa6202c6c1c8578ee69dd22","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"46cb29457af1b267208bce4459af0245262ef4d1ad14c94a0ea2e4ad22bc4095","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"50c252289af604d9dc832800d0b0d6d9e474bee5e4de107b5d0df29194e3c266","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"9cde0add2dcef259d675d7857a2d63c872c0734f52569ee693b1c272cc8cd980","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":9,"runs_affected":5,"failed_runs":5,"run_impact_percent":14.285714285714286,"anchor_occurrences":9,"after_last_push_count":9,"also_in":["DEV","INT","STG"],"trend_counts":[0,3,3,1,1,1,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001307698\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-22-094635, targetMinor=4.22.0-0.nightly-multi-2026-08-23-080550) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-23-080550\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001307698\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-22-094635, targetMinor=4.22.0-0.nightly-multi-2026-08-23-080550) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-23-080550\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000e06ca8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-22-094635, targetMinor=4.22.0-0.nightly-multi-2026-08-21-221014) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-21-221014\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000e06ca8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-22-094635, targetMinor=4.22.0-0.nightly-multi-2026-08-21-221014) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-21-221014\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f0d698\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-20-065923, targetMinor=4.22.0-0.nightly-multi-2026-08-20-215447) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-20-215447\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f0d698\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-20-065923, targetMinor=4.22.0-0.nightly-multi-2026-08-20-215447) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-20-215447\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091661989947379712","occurred_at":"2026-08-23T23:00:54Z","signature_id":"8cf3b387c284e6fef4c388bce44fc9a4f747d49436eee0a48d049c94e88389e0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"a602747a6f8ff912aa52c3897a0845124e7f37239c0849398a58cd662ecde56e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","signature_id":"c65bf0079bcd2ce58e12f45cf9f37bbe8f148f64187938cc3ba61a042251c88d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090574634771550208","occurred_at":"2026-08-20T23:00:08Z","signature_id":"01e9a66c2f609a10ecc530784123db30dcce788426aa44da969016690f7813c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090574634771550208","occurred_at":"2026-08-20T23:00:08Z","signature_id":"0381d0695962715c36dc0eb5609e6d72fe4109b956e9cc97e338fa4cfb93e786","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090574634771550208","occurred_at":"2026-08-20T23:00:08Z","signature_id":"3bc36dba37a0ed6333bc3c086324611c7fcddcc4f56b00a369cafc8b3309aabe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090212357799153664","occurred_at":"2026-08-19T23:00:07Z","signature_id":"11b4a821d08161a4eb2a13a1638065b8dff2c9c2f3883f32f3c6227b7bb9744d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090212357799153664","occurred_at":"2026-08-19T23:00:07Z","signature_id":"2c525ebc161de7eb9503106b010319d214b7fa6e2c59326d7f45c296b91aa5eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090212357799153664","occurred_at":"2026-08-19T23:00:07Z","signature_id":"931c8963596d2fb1aa67e022746279eb97fffd152acbead1983aa67d0c17f754","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6411b2b281a97b7e65400b269143638faa6befdec2dd319ac1327622333be049","failure_pattern":"node pool \u003cnodepool\u003e verification failed: \u003cnode\u003e (no version in containerRuntimeVersion \"cri-o://1","search_query":"node pool upgrade verification failed: o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp (no version in containerRuntimeVersion \"cri-o://1","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":14.285714285714286,"anchor_occurrences":5,"after_last_push_count":5,"also_in":["STG"],"trend_counts":[0,0,1,1,1,1,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002b4c960\u003e: \n    node pool upgrade verification failed: o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp (no version in containerRuntimeVersion \"cri-o://1.35.4\"); o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]}]\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002b4c960\u003e: \n    node pool upgrade verification failed: o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp (no version in containerRuntimeVersion \"cri-o://1.35.4\"); o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]}]\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000515050\u003e: \n    node pool upgrade verification failed: w1h2v3r9e6e0s7b-nps-420-422-x87c9-8vvqf (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w1h2v3r9e6e0s7b-nps-420-422-x87c9-v4xt9 (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w1h2v3r9e6e0s7b-nps-420-422-x87c9-8vvqf\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"w1h2v3r9e6e0s7b-nps-420-422-x87c9-v4xt9\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]}]\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000515050\u003e: \n    node pool upgrade verification failed: w1h2v3r9e6e0s7b-nps-420-422-x87c9-8vvqf (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w1h2v3r9e6e0s7b-nps-420-422-x87c9-v4xt9 (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w1h2v3r9e6e0s7b-nps-420-422-x87c9-8vvqf\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"w1h2v3r9e6e0s7b-nps-420-422-x87c9-v4xt9\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]}]\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000d22130\u003e: \n    node pool upgrade verification failed: w9t9d8g9i9t3j7r-nps-420-422-jxpcn-kdnmh (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w9t9d8g9i9t3j7r-nps-420-422-jxpcn-zmxms (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w9t9d8g9i9t3j7r-nps-420-422-jxpcn-kdnmh\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"w9t9d8g9i9t3j7r-nps-420-422-jxpcn-zmxms\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000d22130\u003e: \n    node pool upgrade verification failed: w9t9d8g9i9t3j7r-nps-420-422-jxpcn-kdnmh (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w9t9d8g9i9t3j7r-nps-420-422-jxpcn-zmxms (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w9t9d8g9i9t3j7r-nps-420-422-jxpcn-kdnmh\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"w9t9d8g9i9t3j7r-nps-420-422-jxpcn-zmxms\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"0c1e40a7688ce333e1802f99689a7431268dce71db699919518e6ab02caaba1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091661989947379712","occurred_at":"2026-08-23T23:00:54Z","signature_id":"e1dc8136aa23907f387f830aa03492aac1f4932daeac05e7f049e4a0362440fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"31d7531de41f020e51232361d7f51aca32428ae38cb8e072eb49e6914e64342e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","signature_id":"f9145ac5e2884f8eb9ccef988d1f85c4fe55e8c0ff9c9f95d10d84fcc0da9011","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090574634771550208","occurred_at":"2026-08-20T23:00:08Z","signature_id":"7648865fe6b91dd3936801eb2fb5190babe872afc58659008e03f5db8190bdd7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"06545dfa74499fb7ed3922d254aeb9be3949de04bfba27647afd1af0e1481ab3","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":5,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.714285714285714,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,5,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-19T07:04:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000817120\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9k28vzcm69z8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d038a282-6d26-4349-9d6e-fafe88a9f437\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d038a282-6d26-4349-9d6e-fafe88a9f437\",\n      \"name\": \"d038a282-6d26-4349-9d6e-fafe88a9f437\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:17:50.18783504Z\",\n      \"endTime\": \"2026-08-19T07:36:53.392353745Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000817120\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9k28vzcm69z8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d038a282-6d26-4349-9d6e-fafe88a9f437\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d038a282-6d26-4349-9d6e-fafe88a9f437\",\n      \"name\": \"d038a282-6d26-4349-9d6e-fafe88a9f437\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:17:50.18783504Z\",\n      \"endTime\": \"2026-08-19T07:36:53.392353745Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-dvgp2ljncmzv/cluster-candidate-4-21-m56pdm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000866520\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-m56pdm: failed waiting for cluster=\"cluster-candidate-4-21-m56pdm\" in resourcegroup=\"rg-candidate-4-21-dvgp2ljncmzv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc9b40e4-dff9-45e8-b9ce-61a92321163f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc9b40e4-dff9-45e8-b9ce-61a92321163f\",\n      \"name\": \"dc9b40e4-dff9-45e8-b9ce-61a92321163f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:48:53.852097725Z\",\n      \"endTime\": \"2026-08-19T08:07:54.416992123Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-dvgp2ljncmzv/cluster-candidate-4-21-m56pdm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000866520\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-m56pdm: failed waiting for cluster=\"cluster-candidate-4-21-m56pdm\" in resourcegroup=\"rg-candidate-4-21-dvgp2ljncmzv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc9b40e4-dff9-45e8-b9ce-61a92321163f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc9b40e4-dff9-45e8-b9ce-61a92321163f\",\n      \"name\": \"dc9b40e4-dff9-45e8-b9ce-61a92321163f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:48:53.852097725Z\",\n      \"endTime\": \"2026-08-19T08:07:54.416992123Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f00ec0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-6k6scsrzs89w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/002cf859-3aea-4235-ade3-bfe0bd472945\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/002cf859-3aea-4235-ade3-bfe0bd472945\",\n      \"name\": \"002cf859-3aea-4235-ade3-bfe0bd472945\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:46:18.55837028Z\",\n      \"endTime\": \"2026-08-19T03:05:22.599243963Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f00ec0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-6k6scsrzs89w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/002cf859-3aea-4235-ade3-bfe0bd472945\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/002cf859-3aea-4235-ade3-bfe0bd472945\",\n      \"name\": \"002cf859-3aea-4235-ade3-bfe0bd472945\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:46:18.55837028Z\",\n      \"endTime\": \"2026-08-19T03:05:22.599243963Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089972004693217280","occurred_at":"2026-08-19T07:04:47Z","signature_id":"085089ac4054e8181d0c051218fad6ee05880cc9bf4ac62d6c057d5a72ced7e0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089972004693217280","occurred_at":"2026-08-19T07:04:47Z","signature_id":"4b40e63cf83ed42345aea20290052f9328495d4bd373d02b0972bffda86f67ae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"02267e9543b2b9550b0b8ec4e4509332326305957c8b1b2cc277d4e59cfe2ac0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"1912b82770035cf45f0662fae4bd7f0a6137993a97ddf66c67b0b086d6cf265f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"c54cac1173c067f540664321c09457b751be9c7b0afb8567c53a6ded23cc7177","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"beaf47bd37dc21f47808a0240288ea4322da110d92ca67bf08de0f6793353753","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.20\": must be at least 4.20.35; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":11.428571428571429,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,1,1,1,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-2m6hjk\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00144a750\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-6dbxgwtzxfsh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-2m6hjk\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-2m6hjk\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00144a750\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-6dbxgwtzxfsh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-2m6hjk\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-96fcvx\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008e5080\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-qkzx8x2fwwlv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-96fcvx\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-96fcvx\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008e5080\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-qkzx8x2fwwlv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-96fcvx\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-n9x84d\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001989aa0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-c8sxt5rwcs2x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-n9x84d\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-n9x84d\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001989aa0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-c8sxt5rwcs2x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-n9x84d\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2092069681539059712","occurred_at":"2026-08-25T02:00:55Z","signature_id":"6b7decf97f628a3a86f9cc81aac7b2b793ce71dbec005f9c0bbc92ad65ed2200","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"67be8c6caed95a193a1745532d025891093937507a496524f899a7d2c80f89f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091344896685248512","occurred_at":"2026-08-23T02:00:53Z","signature_id":"7fadbb50104905453b171581a3b547728bfc635ffabfe3f306a6856ceb68c169","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090982470148165632","occurred_at":"2026-08-22T02:00:44Z","signature_id":"82b312bb71915f291931b82d9a4f4b5450769045c0f3ebb03733360bf9b57bf0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5443585aedc88f5d2589a8fb3795e77266206bcfdb680d3cfbedccaf7bc67f50","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21\": must be at least 4.21.30; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":11.428571428571429,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,1,1,1,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-hc729b\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000cc7e30\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-4j8dsngwmfpg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-hc729b\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-hc729b\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000cc7e30\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-4j8dsngwmfpg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-hc729b\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-znvqfb\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008e8630\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-bb24w6nmgn9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-znvqfb\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-znvqfb\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008e8630\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-bb24w6nmgn9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-znvqfb\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-jjmbhq\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001168f00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-zv8kjsp968km/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-jjmbhq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-jjmbhq\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001168f00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-zv8kjsp968km/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-jjmbhq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2092069681539059712","occurred_at":"2026-08-25T02:00:55Z","signature_id":"47e6560075ab2811dc4afc63d80fcd866f7a8dbf54b0f240501f8704720bfe02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"eabd42ab0d34b7f1d8ca082eb227c6c5a44990af4cb2ad49ebe668cc8663e6ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091344896685248512","occurred_at":"2026-08-23T02:00:53Z","signature_id":"4a64d043a2a51d28eb3268662ea59869b9f2d4cb03e6c4b5e7e27a8633164231","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090982470148165632","occurred_at":"2026-08-22T02:00:44Z","signature_id":"e1ddce92e347e799f32b65d9f1cb530e81c0d6ecb9fa3b0a4f668a226890dd2e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"2b6de8e3eeb7b1eb9cb9733a09ec9f907952649e2f47b910e6bb29761a829864","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.22\": must be at least 4.22.10; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":11.428571428571429,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,1,1,1,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-qp64bq\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a2bec0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-dqpzmwzqshp6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-qp64bq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-qp64bq\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a2bec0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-dqpzmwzqshp6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-qp64bq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-gmlx25\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000703a40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-b6dm7hck8ksd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-gmlx25\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-gmlx25\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000703a40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-b6dm7hck8ksd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-gmlx25\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-7kr5sh\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00155f380\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-85cmmktbjpd6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-7kr5sh\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-7kr5sh\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00155f380\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-85cmmktbjpd6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-7kr5sh\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2092069681539059712","occurred_at":"2026-08-25T02:00:55Z","signature_id":"5f57af3b362f707a2fd214f4c7abae7f575a8b66d80b93974a89ebabf4108803","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"233817672ec9f15c61036dd69be2232e5da454215c4ae03262d5d9d1147fe964","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091344896685248512","occurred_at":"2026-08-23T02:00:53Z","signature_id":"e2599145a605851478085e130e5b7fbbe8ead4f7a954be8d2bd8955c7c63ea0b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090982470148165632","occurred_at":"2026-08-22T02:00:44Z","signature_id":"5751bb61a46186e35d46b5f3aa2658a3a2eb89b66543f8c048fbecc5ffa16bc7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6c0df54cac07833305ae54d7a30bcf9a55390b618040466bd5f2843d80e698a8","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":11.428571428571429,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,1,1,1,1],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000b1c2d0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000b1c2d0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000ad6720\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000ad6720\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000cde978\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000cde978\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2092069681539059712","occurred_at":"2026-08-25T02:00:55Z","signature_id":"328ab83e407af59a94692dc615a105004e707662f379710e72fd8156cdefd247","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"3fd31e6b902a75b339639f3cb28f3e152b295a16939ec2f7056ea2da7a8f4e73","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091344896685248512","occurred_at":"2026-08-23T02:00:53Z","signature_id":"4708ea587292d39020c8eb2261a75c138a9781a6caaa1b7c86a80d70fb0d8b1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090982470148165632","occurred_at":"2026-08-22T02:00:44Z","signature_id":"7171b853a0dfa100dae5cff129272db18b20b0216e77e4c1ba95edf50cfab602","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6e0339db7bd54e6d51e0a052774819897d4aae43976b41e9f3679271ea556d4f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-21-184859 to 5.1.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":8.571428571428571,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,1,1,1,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-wzcn8d\" to 5.1.0-0.nightly-multi-2026-08-21-161733\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001250e60\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-wzcn8d\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-6sv96bxs684h\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/28f12e91-44fb-4368-818f-606f837a5513\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/28f12e91-44fb-4368-818f-606f837a5513\",\n      \"name\": \"28f12e91-44fb-4368-818f-606f837a5513\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T23:47:34.702003558Z\",\n      \"endTime\": \"2026-08-23T23:47:35.713542837Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-21-184859 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-wzcn8d\" to 5.1.0-0.nightly-multi-2026-08-21-161733\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001250e60\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-wzcn8d\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-6sv96bxs684h\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/28f12e91-44fb-4368-818f-606f837a5513\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/28f12e91-44fb-4368-818f-606f837a5513\",\n      \"name\": \"28f12e91-44fb-4368-818f-606f837a5513\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T23:47:34.702003558Z\",\n      \"endTime\": \"2026-08-23T23:47:35.713542837Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-21-184859 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-8pb9fk\" to 5.1.0-0.nightly-multi-2026-08-21-161733\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00162c920\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-8pb9fk\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-rvbnzc42snz2\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/983e4736-d961-45bf-8e63-27df9f4392b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/983e4736-d961-45bf-8e63-27df9f4392b0\",\n      \"name\": \"983e4736-d961-45bf-8e63-27df9f4392b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:48:09.603825759Z\",\n      \"endTime\": \"2026-08-22T23:48:09.987134271Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-21-184859 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-8pb9fk\" to 5.1.0-0.nightly-multi-2026-08-21-161733\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00162c920\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-8pb9fk\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-rvbnzc42snz2\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/983e4736-d961-45bf-8e63-27df9f4392b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/983e4736-d961-45bf-8e63-27df9f4392b0\",\n      \"name\": \"983e4736-d961-45bf-8e63-27df9f4392b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:48:09.603825759Z\",\n      \"endTime\": \"2026-08-22T23:48:09.987134271Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-21-184859 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-62zzff\" to 5.1.0-0.nightly-multi-2026-08-21-161733\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00102a5c0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-62zzff\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-stlzppcpw4zq\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b61f43e-2185-4f9f-9b05-3ed88171c66f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b61f43e-2185-4f9f-9b05-3ed88171c66f\",\n      \"name\": \"1b61f43e-2185-4f9f-9b05-3ed88171c66f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:44:09.672524927Z\",\n      \"endTime\": \"2026-08-21T23:44:17.756539054Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-21-184859 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-62zzff\" to 5.1.0-0.nightly-multi-2026-08-21-161733\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00102a5c0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-62zzff\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-stlzppcpw4zq\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b61f43e-2185-4f9f-9b05-3ed88171c66f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b61f43e-2185-4f9f-9b05-3ed88171c66f\",\n      \"name\": \"1b61f43e-2185-4f9f-9b05-3ed88171c66f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:44:09.672524927Z\",\n      \"endTime\": \"2026-08-21T23:44:17.756539054Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-21-184859 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091661989947379712","occurred_at":"2026-08-23T23:00:54Z","signature_id":"eaa551a9dd1eac38a1c62130fb5c3e51993a843414977ead9e55a0b291b04a09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"f374e19a5b7a4b8d52f77129b3b153c431fbc68fe03a795ff29942859e5e3fe4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","signature_id":"e3efa032fccb6cd516133bd1ad89ebb60e8dae5741cd61caf0ccbc55f84ea58e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\", caused by: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-rm9wbpvfcdwm, error: context deadline exceeded, error: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-rm9wbpvfcdwm, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":3,"after_last_push_count":4,"also_in":["DEV"],"trend_counts":[0,3,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-19T05:29:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00177ecc0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"pullsecret-test-7k8vgxjbrndr\", caused by: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7k8vgxjbrndr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-7k8vgxjbrndr, error: context deadline exceeded, error: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7k8vgxjbrndr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-7k8vgxjbrndr, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00177ecc0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"pullsecret-test-7k8vgxjbrndr\", caused by: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7k8vgxjbrndr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-7k8vgxjbrndr, error: context deadline exceeded, error: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7k8vgxjbrndr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-7k8vgxjbrndr, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f46198\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\", caused by: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-rm9wbpvfcdwm, error: context deadline exceeded, error: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-rm9wbpvfcdwm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f46198\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\", caused by: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-rm9wbpvfcdwm, error: context deadline exceeded, error: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-rm9wbpvfcdwm, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000cdc768\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"oidc-wi-4m2nc64cmj8g\", caused by: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-4m2nc64cmj8g\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-4m2nc64cmj8g, error: context deadline exceeded, error: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-4m2nc64cmj8g\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-4m2nc64cmj8g, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000cdc768\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"oidc-wi-4m2nc64cmj8g\", caused by: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-4m2nc64cmj8g\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-4m2nc64cmj8g, error: context deadline exceeded, error: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-4m2nc64cmj8g\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-4m2nc64cmj8g, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"3df638fe5d43cca0e2f4e149ed492e56b8e1203df35cc7e58fdae4f9e245b884","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"4a223b1d651f299719cd9b6e11edb5f3c3501b11271b66fb68c6008a86d9bda9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"888f561b7f7383f42d7aaa8d1c30b4ffa60e6f2c0e4c3eba34f35a033b8de56c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3d8b21ba20484aaa0ba81955928ba0600e83c5e3854c628e01a8d708233935f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":3,"after_last_push_count":5,"also_in":["STG"],"trend_counts":[0,3,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-19T05:29:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000be4420\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-wsd5rv\" in resourcegroup=\"rg-np-skip-minor-wsd5rv-7zc5n446c49n\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/44029b45-2a06-403c-ad77-6b306bc07eac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/44029b45-2a06-403c-ad77-6b306bc07eac\",\n      \"name\": \"44029b45-2a06-403c-ad77-6b306bc07eac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:45:10.619596164Z\",\n      \"endTime\": \"2026-08-19T08:09:15.782431257Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9gasbeacg14154vgqri92i9qj3lj2u still exists (deletion dispatched at 2026-08-19T07:45:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000be4420\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-wsd5rv\" in resourcegroup=\"rg-np-skip-minor-wsd5rv-7zc5n446c49n\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/44029b45-2a06-403c-ad77-6b306bc07eac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/44029b45-2a06-403c-ad77-6b306bc07eac\",\n      \"name\": \"44029b45-2a06-403c-ad77-6b306bc07eac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:45:10.619596164Z\",\n      \"endTime\": \"2026-08-19T08:09:15.782431257Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9gasbeacg14154vgqri92i9qj3lj2u still exists (deletion dispatched at 2026-08-19T07:45:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f06630\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-dg-minor-zm2mgg\" in resourcegroup=\"rg-np-dg-minor-zm2mgg-fwcclg2fsc25\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/655cfcac-fbdc-4612-9646-a9b1d1f8165f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/655cfcac-fbdc-4612-9646-a9b1d1f8165f\",\n      \"name\": \"655cfcac-fbdc-4612-9646-a9b1d1f8165f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:45:55.316555912Z\",\n      \"endTime\": \"2026-08-19T08:09:55.778621843Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ge0h765335a8886qbg3l13s24maom still exists (deletion dispatched at 2026-08-19T07:45:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f06630\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-dg-minor-zm2mgg\" in resourcegroup=\"rg-np-dg-minor-zm2mgg-fwcclg2fsc25\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/655cfcac-fbdc-4612-9646-a9b1d1f8165f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/655cfcac-fbdc-4612-9646-a9b1d1f8165f\",\n      \"name\": \"655cfcac-fbdc-4612-9646-a9b1d1f8165f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:45:55.316555912Z\",\n      \"endTime\": \"2026-08-19T08:09:55.778621843Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ge0h765335a8886qbg3l13s24maom still exists (deletion dispatched at 2026-08-19T07:45:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00116e1b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-dxqx7n\" in resourcegroup=\"rg-np-version-upgrade-dxqx7n-j7h8sl7k4nkw\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6d75e2fa-2f17-4c70-b06f-914c5a7c9ff5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6d75e2fa-2f17-4c70-b06f-914c5a7c9ff5\",\n      \"name\": \"6d75e2fa-2f17-4c70-b06f-914c5a7c9ff5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:31:53.195424705Z\",\n      \"endTime\": \"2026-08-19T07:55:54.488364404Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9g68gmftv667t18n90kkemktfaa0e8 still exists (deletion dispatched at 2026-08-19T07:31:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00116e1b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-dxqx7n\" in resourcegroup=\"rg-np-version-upgrade-dxqx7n-j7h8sl7k4nkw\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6d75e2fa-2f17-4c70-b06f-914c5a7c9ff5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6d75e2fa-2f17-4c70-b06f-914c5a7c9ff5\",\n      \"name\": \"6d75e2fa-2f17-4c70-b06f-914c5a7c9ff5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:31:53.195424705Z\",\n      \"endTime\": \"2026-08-19T07:55:54.488364404Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9g68gmftv667t18n90kkemktfaa0e8 still exists (deletion dispatched at 2026-08-19T07:31:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"88671d66208c0e2cca5412ad058e14717aa66730bf52487d7d3fe6d9e02a93d7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"95e98ccacc154ed064ebee4e99e352acd3e83cfe126b57dd54db13d41e2c9d3b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"e14ae40adcdf76d950a7f4a6d2404b93325e238158216943514be2323a70b355","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.714285714285714,"anchor_occurrences":2,"after_last_push_count":13,"also_in":["DEV","STG"],"trend_counts":[0,1,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":20,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-20 with version 4.20.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000706700\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7ts9cp\" in resourcegroup=\"rg-np-version-upgrade-7ts9cp-bztnq2hvf6sh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6c93bda8-add1-4e48-9eea-0b0a433d2308\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6c93bda8-add1-4e48-9eea-0b0a433d2308\",\n      \"name\": \"6c93bda8-add1-4e48-9eea-0b0a433d2308\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:54:35.924673301Z\",\n      \"endTime\": \"2026-08-21T03:13:35.571507889Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-20 with version 4.20.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000706700\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7ts9cp\" in resourcegroup=\"rg-np-version-upgrade-7ts9cp-bztnq2hvf6sh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6c93bda8-add1-4e48-9eea-0b0a433d2308\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6c93bda8-add1-4e48-9eea-0b0a433d2308\",\n      \"name\": \"6c93bda8-add1-4e48-9eea-0b0a433d2308\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:54:35.924673301Z\",\n      \"endTime\": \"2026-08-21T03:13:35.571507889Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f5dc80\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-sj5szv8t6s8s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37c1aadb-f927-4c30-95d8-892451317321\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37c1aadb-f927-4c30-95d8-892451317321\",\n      \"name\": \"37c1aadb-f927-4c30-95d8-892451317321\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T00:43:05.670558184Z\",\n      \"endTime\": \"2026-08-20T01:02:11.662741691Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f5dc80\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-sj5szv8t6s8s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37c1aadb-f927-4c30-95d8-892451317321\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37c1aadb-f927-4c30-95d8-892451317321\",\n      \"name\": \"37c1aadb-f927-4c30-95d8-892451317321\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T00:43:05.670558184Z\",\n      \"endTime\": \"2026-08-20T01:02:11.662741691Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","signature_id":"53444ab8124f99545024afe8c22d658c5922fe471870c6bfb72c71ce7c7d2fd9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2090221146522783744","occurred_at":"2026-08-19T23:35:30Z","signature_id":"230aa160a5c0cd54b1bd26585190eb1e4882b1ddf6c5769008ac315b46852a77","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"724e7a51f2cd05d0f0c351e24f28344ba4eb40490e6de7ed02ace0e7a7856472","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.714285714285714,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-lrdz8v\" to 4.23.0-0.nightly-multi-2026-08-24-110923\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000881340\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-lrdz8v\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-t25q8kvqvdhw\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\",\n      \"name\": \"4393d587-2d85-4674-a4f6-155ceb1138f4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:48:28.850341951Z\",\n      \"endTime\": \"2026-08-24T23:48:30.944709051Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-lrdz8v\" to 4.23.0-0.nightly-multi-2026-08-24-110923\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000881340\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-lrdz8v\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-t25q8kvqvdhw\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\",\n      \"name\": \"4393d587-2d85-4674-a4f6-155ceb1138f4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:48:28.850341951Z\",\n      \"endTime\": \"2026-08-24T23:48:30.944709051Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-fld59h\" to 4.23.0-0.nightly-multi-2026-08-22-063717\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004b76c0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-fld59h\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-rjgd8k2jwrfb\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f8fb992a-e4cf-48ed-a1c6-8971aa44e1fa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f8fb992a-e4cf-48ed-a1c6-8971aa44e1fa\",\n      \"name\": \"f8fb992a-e4cf-48ed-a1c6-8971aa44e1fa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T23:47:13.066383422Z\",\n      \"endTime\": \"2026-08-23T23:47:20.544658196Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-fld59h\" to 4.23.0-0.nightly-multi-2026-08-22-063717\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004b76c0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-fld59h\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-rjgd8k2jwrfb\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f8fb992a-e4cf-48ed-a1c6-8971aa44e1fa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f8fb992a-e4cf-48ed-a1c6-8971aa44e1fa\",\n      \"name\": \"f8fb992a-e4cf-48ed-a1c6-8971aa44e1fa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T23:47:13.066383422Z\",\n      \"endTime\": \"2026-08-23T23:47:20.544658196Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"6663bc04df6e96cb243dd9f35fd825b118edab670cd360f1ba5fc79ade7a0f44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091661989947379712","occurred_at":"2026-08-23T23:00:54Z","signature_id":"3949a84ac57a39a215e47f5d92006980597324fda0c46e302caaa5b9ad11956f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"816d4adf1b0cead0de869e12fb780cc1877997e002f751655ae6980e24a728c2","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.20\": must be at least 4.20.34; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.714285714285714,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["STG"],"trend_counts":[0,0,1,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-7kjkxp\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001506ab0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-rncmtdm68xg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-7kjkxp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-7kjkxp\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001506ab0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-rncmtdm68xg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-7kjkxp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-fhgsjh\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000ad93e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-khb5brts9d48/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-fhgsjh\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-fhgsjh\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000ad93e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-khb5brts9d48/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-fhgsjh\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","signature_id":"6b18799e0c02120baa6b7406c38ed69946bfb8f075f9525de85fb5d61e35a451","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090257540720889856","occurred_at":"2026-08-20T02:00:07Z","signature_id":"11b69fdcc52114ea6507f2ce22ea5cab6804563d71d415697657f4abc51fb821","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4afd6a1ada377507e7e7e6c2a964127b8acdde371be3fb4961b6586bdd4a43de","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21\": must be at least 4.21.29; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.714285714285714,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["STG"],"trend_counts":[0,0,1,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-9hq8tp\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000c1fa10\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-dltt2s2c9z4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-9hq8tp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-9hq8tp\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000c1fa10\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-dltt2s2c9z4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-9hq8tp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-4bcwd7\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001318120\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-dcr28mz4j9bb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-4bcwd7\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-4bcwd7\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001318120\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-dcr28mz4j9bb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-4bcwd7\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","signature_id":"e2e437dfbe2809e1622aea345f8304dc17ed03f4c4745b55ee016489aa7dc065","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090257540720889856","occurred_at":"2026-08-20T02:00:07Z","signature_id":"e53f4eb7b6705b82dd48b0f763f426c94d8559f03442eb10e9c7be98a65128e5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"41121f687ee586823386c215e67a10fa5622aef7585d34e6556590734bdccec3","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.22\": must be at least 4.22.9; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.714285714285714,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["STG"],"trend_counts":[0,0,1,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-d9vwt2\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001576c60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-bmvjhbv2q2z7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-d9vwt2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-d9vwt2\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001576c60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-bmvjhbv2q2z7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-d9vwt2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-cd76bv\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012a2480\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-5z4mq62hpnnt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-cd76bv\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-cd76bv\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012a2480\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-5z4mq62hpnnt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-cd76bv\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","signature_id":"b0896529db37c90b279c53d6fc422ea9199968c1d2dd406897213a62eda70455","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090257540720889856","occurred_at":"2026-08-20T02:00:07Z","signature_id":"dd299017d2dd3de6a3a73aeae80d69921f52513a63912d73b2ece7bda43e062d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dcad60eb8db626db10bfb0812a18e7c392049987a81c4fcbcdc5140e0032dc41","failure_pattern":"Cluster should become ready within '20.000000' minutes","search_query":"Cluster should become ready within '20.000000' minutes","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T08:22:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:163]: Timed out after 1200.003s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:163]: Timed out after 1200.003s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"694e2847365ce62823438330cc701f9592c480ebed5f88d5217b25baa06ae6c6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5fd9534fdae604c74e3a3a42c818683431299c230e99d1c0094cdeba9bca67ef","failure_pattern":"ERROR CODE: Conflict; detail message Cannot update resource while resource is \"provisioning\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T08:22:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:314]: failed to create HCP cluster np-noedge-chjtcz\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fcc1e0\u003e: \n    failed to create HCP cluster np-noedge-chjtcz: failed starting cluster creation \"np-noedge-chjtcz\" in resourcegroup=\"rg-np-noedge-chjtcz-4lgwg5nz5hfm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-chjtcz-4lgwg5nz5hfm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-chjtcz\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"provisioning\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-chjtcz-4lgwg5nz5hfm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-chjtcz\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:314]: failed to create HCP cluster np-noedge-chjtcz\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fcc1e0\u003e: \n    failed to create HCP cluster np-noedge-chjtcz: failed starting cluster creation \"np-noedge-chjtcz\" in resourcegroup=\"rg-np-noedge-chjtcz-4lgwg5nz5hfm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-chjtcz-4lgwg5nz5hfm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-chjtcz\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"provisioning\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-chjtcz-4lgwg5nz5hfm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-chjtcz\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"6fed47754c392741faea417d75c6c36322e9e06425428e55e56bbee0033699e2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5850eea44ccf20fbe9121c0cac3952957d3152f6db0ef54a778fb6e1a4f24903","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OK","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T05:29:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:79]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a92d60\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-n784sz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-n784sz/operationStatuses/08584144855033284603\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"OK\",\n            \"message\": \"\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:79]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a92d60\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-n784sz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-n784sz/operationStatuses/08584144855033284603\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"OK\",\n            \"message\": \"\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"002dc1bd12014246737866976762e493ca1bc3b77a01cecd7b6242c9e5c4054e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"0438873facb9dd98e74695c849423f31b775588ec5b0ef7ead10a8182c141d31","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":12,"also_in":["DEV","STG"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-19T07:04:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00082a120\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-ml5crr8fbhl2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eceba15d-5a1a-4b27-a4e7-b24724b60536\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eceba15d-5a1a-4b27-a4e7-b24724b60536\",\n      \"name\": \"eceba15d-5a1a-4b27-a4e7-b24724b60536\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:18:43.866677598Z\",\n      \"endTime\": \"2026-08-19T07:37:43.126289194Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00082a120\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-ml5crr8fbhl2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eceba15d-5a1a-4b27-a4e7-b24724b60536\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eceba15d-5a1a-4b27-a4e7-b24724b60536\",\n      \"name\": \"eceba15d-5a1a-4b27-a4e7-b24724b60536\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:18:43.866677598Z\",\n      \"endTime\": \"2026-08-19T07:37:43.126289194Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089972004693217280","occurred_at":"2026-08-19T07:04:47Z","signature_id":"acc605509df755de66458b8411f8ea9e7693a79d546c6990432cabd1a2177e62","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"cb8ca43a79863985fe5fa47ef1bbe80d7bc25200d089461928007bb1c0bcd82f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":9,"also_in":["DEV","STG"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-19T02:00:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-zjc9p9k8fp8k/cluster-candidate-5-0-kz87c7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9e720\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-kz87c7: failed waiting for cluster=\"cluster-candidate-5-0-kz87c7\" in resourcegroup=\"rg-candidate-5-0-zjc9p9k8fp8k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a3081846-fb1d-49f7-9bc5-d276f1e7c8f7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a3081846-fb1d-49f7-9bc5-d276f1e7c8f7\",\n      \"name\": \"a3081846-fb1d-49f7-9bc5-d276f1e7c8f7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:38:11.46423025Z\",\n      \"endTime\": \"2026-08-19T02:57:21.467415333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-zjc9p9k8fp8k/cluster-candidate-5-0-kz87c7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9e720\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-kz87c7: failed waiting for cluster=\"cluster-candidate-5-0-kz87c7\" in resourcegroup=\"rg-candidate-5-0-zjc9p9k8fp8k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a3081846-fb1d-49f7-9bc5-d276f1e7c8f7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a3081846-fb1d-49f7-9bc5-d276f1e7c8f7\",\n      \"name\": \"a3081846-fb1d-49f7-9bc5-d276f1e7c8f7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:38:11.46423025Z\",\n      \"endTime\": \"2026-08-19T02:57:21.467415333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"51e876eeeb423bd303d9c668183a1e9829a3464504c6584d4d08f6a6cd7ae4c6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"df07fa0d010168626c38c9ee8c7f0b6209b1dfc859d2a8f4c53ac9f63886cbbe","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","STG"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T02:00:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-20-fkndtwz8x29s/cluster-candidate-4-20-kpswjg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c0ef80\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-kpswjg: failed waiting for cluster=\"cluster-candidate-4-20-kpswjg\" in resourcegroup=\"rg-candidate-4-20-fkndtwz8x29s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5ce8c6d6-e8b9-4d76-97a6-d8ba04a44b30\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5ce8c6d6-e8b9-4d76-97a6-d8ba04a44b30\",\n      \"name\": \"5ce8c6d6-e8b9-4d76-97a6-d8ba04a44b30\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:36:44.929925762Z\",\n      \"endTime\": \"2026-08-19T02:55:50.765263478Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-20-fkndtwz8x29s/cluster-candidate-4-20-kpswjg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c0ef80\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-kpswjg: failed waiting for cluster=\"cluster-candidate-4-20-kpswjg\" in resourcegroup=\"rg-candidate-4-20-fkndtwz8x29s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5ce8c6d6-e8b9-4d76-97a6-d8ba04a44b30\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5ce8c6d6-e8b9-4d76-97a6-d8ba04a44b30\",\n      \"name\": \"5ce8c6d6-e8b9-4d76-97a6-d8ba04a44b30\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:36:44.929925762Z\",\n      \"endTime\": \"2026-08-19T02:55:50.765263478Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"88a3c1d52823c5aea1fc6c80298693b077a17e7246695e45a14c8bda69b5b204","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3148230ac2b952709eb29676f2aa7a717a33aa306cef8cd69c888b564066382e","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","INT","STG"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T04:55:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e46720\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-mj9b89zp6qrs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/764a11ed-2f54-474f-bdce-7ac64c3a7a20\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/764a11ed-2f54-474f-bdce-7ac64c3a7a20\",\n      \"name\": \"764a11ed-2f54-474f-bdce-7ac64c3a7a20\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T05:36:20.303691403Z\",\n      \"endTime\": \"2026-08-19T05:55:29.476993829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e46720\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-mj9b89zp6qrs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/764a11ed-2f54-474f-bdce-7ac64c3a7a20\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/764a11ed-2f54-474f-bdce-7ac64c3a7a20\",\n      \"name\": \"764a11ed-2f54-474f-bdce-7ac64c3a7a20\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T05:36:20.303691403Z\",\n      \"endTime\": \"2026-08-19T05:55:29.476993829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089939212420780032","occurred_at":"2026-08-19T04:55:12Z","signature_id":"8e433d3376cdbe8b927b5bd34a27c2e303c2977b118596635ec3afc2173c7b6f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"87342952322b831799ec55685129d31670bbfdbd908032dee3aa610155e46b8d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T07:04:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f896e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fh79ld4nbj75\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5bea30a8-9570-45db-b908-4c4e07deca9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5bea30a8-9570-45db-b908-4c4e07deca9b\",\n      \"name\": \"5bea30a8-9570-45db-b908-4c4e07deca9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:16:30.019566976Z\",\n      \"endTime\": \"2026-08-19T07:35:32.987525861Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f896e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fh79ld4nbj75\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5bea30a8-9570-45db-b908-4c4e07deca9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5bea30a8-9570-45db-b908-4c4e07deca9b\",\n      \"name\": \"5bea30a8-9570-45db-b908-4c4e07deca9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:16:30.019566976Z\",\n      \"endTime\": \"2026-08-19T07:35:32.987525861Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089972004693217280","occurred_at":"2026-08-19T07:04:47Z","signature_id":"93b46a6f12f4806ddb007892005a66398adc1f857b40625a31958a9198b118aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"67e9146bba6a263169b2663e0ece2eff489af41d3e74f4e915f7ce1b74288068","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":12,"also_in":["DEV","INT","STG"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001088a60\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9pxxtrlstzqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69f46a0a-74d3-4dcf-bf34-2997e6551fc7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69f46a0a-74d3-4dcf-bf34-2997e6551fc7\",\n      \"name\": \"69f46a0a-74d3-4dcf-bf34-2997e6551fc7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:06:00.903480575Z\",\n      \"endTime\": \"2026-08-21T02:25:03.149034019Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001088a60\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9pxxtrlstzqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69f46a0a-74d3-4dcf-bf34-2997e6551fc7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69f46a0a-74d3-4dcf-bf34-2997e6551fc7\",\n      \"name\": \"69f46a0a-74d3-4dcf-bf34-2997e6551fc7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:06:00.903480575Z\",\n      \"endTime\": \"2026-08-21T02:25:03.149034019Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","signature_id":"34afd7909da6ce5cf526eb6e72eebbe525ca7ddbcc6a4275c8538eb2c2fc8879","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79d42a294edfd7eaa19516eb5e39cb2fbbc57c83345983aea3a63ac716c4f9fa","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["DEV","INT","STG"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-19T02:00:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000923f40\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-886kmmqnwq8r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a873ac5e-6930-421b-80c5-b0073727d292\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a873ac5e-6930-421b-80c5-b0073727d292\",\n      \"name\": \"a873ac5e-6930-421b-80c5-b0073727d292\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:03:10.761261438Z\",\n      \"endTime\": \"2026-08-19T02:22:19.897779073Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000923f40\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-886kmmqnwq8r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a873ac5e-6930-421b-80c5-b0073727d292\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a873ac5e-6930-421b-80c5-b0073727d292\",\n      \"name\": \"a873ac5e-6930-421b-80c5-b0073727d292\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:03:10.761261438Z\",\n      \"endTime\": \"2026-08-19T02:22:19.897779073Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"ef1bb3e717af80552bc856ab57b6fe7465a664fd7db1d089b9b6a66971694d25","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6ae125442940bde90789cccefefed6da7645f30b5d4ff58e7803d4ddb14827ec","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T07:04:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00161c860\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-x98pjvvxgzk4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/03e64f83-5bd0-4f0b-819b-4b88633c17cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/03e64f83-5bd0-4f0b-819b-4b88633c17cf\",\n      \"name\": \"03e64f83-5bd0-4f0b-819b-4b88633c17cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:16:18.329159963Z\",\n      \"endTime\": \"2026-08-19T07:35:23.12849681Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00161c860\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-x98pjvvxgzk4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/03e64f83-5bd0-4f0b-819b-4b88633c17cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/03e64f83-5bd0-4f0b-819b-4b88633c17cf\",\n      \"name\": \"03e64f83-5bd0-4f0b-819b-4b88633c17cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:16:18.329159963Z\",\n      \"endTime\": \"2026-08-19T07:35:23.12849681Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089972004693217280","occurred_at":"2026-08-19T07:04:47Z","signature_id":"63c75abc2f7d924366fc2bd1819a4197f85221191693b9ed0f020de2cfde66eb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1f1ac3bd2f30331d5cb9d90f9185c916aceed97388936aff1c3323c7c86be085","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T02:24:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00091a9c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-nzcg89cnmszj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/99a00b91-6d0c-48a1-9fdd-2970d104a1e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/99a00b91-6d0c-48a1-9fdd-2970d104a1e6\",\n      \"name\": \"99a00b91-6d0c-48a1-9fdd-2970d104a1e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:42:34.622468609Z\",\n      \"endTime\": \"2026-08-19T03:01:41.644884055Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00091a9c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-nzcg89cnmszj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/99a00b91-6d0c-48a1-9fdd-2970d104a1e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/99a00b91-6d0c-48a1-9fdd-2970d104a1e6\",\n      \"name\": \"99a00b91-6d0c-48a1-9fdd-2970d104a1e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:42:34.622468609Z\",\n      \"endTime\": \"2026-08-19T03:01:41.644884055Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089901294310395904","occurred_at":"2026-08-19T02:24:31Z","signature_id":"2f15a24b405efc2994292058653832d4b3ff19b68c4d62f2c5ef6861a750fb7f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9f0d2f8bd3c175c3be0acb5a8e8fbae146964e77da19031276e4cea79dffa081","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["DEV","INT","STG"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-19T05:29:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011f0378\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-dw5mg9lsjg9t\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/52d1d0b5-d84a-422f-adff-632e7b15bd86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/52d1d0b5-d84a-422f-adff-632e7b15bd86\",\n      \"name\": \"52d1d0b5-d84a-422f-adff-632e7b15bd86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:21:20.266749938Z\",\n      \"endTime\": \"2026-08-19T07:45:24.43604152Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9g587oc7gqeeckl8k5s1283accvgo6 still exists (deletion dispatched at 2026-08-19T07:21:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011f0378\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-dw5mg9lsjg9t\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/52d1d0b5-d84a-422f-adff-632e7b15bd86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/52d1d0b5-d84a-422f-adff-632e7b15bd86\",\n      \"name\": \"52d1d0b5-d84a-422f-adff-632e7b15bd86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:21:20.266749938Z\",\n      \"endTime\": \"2026-08-19T07:45:24.43604152Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9g587oc7gqeeckl8k5s1283accvgo6 still exists (deletion dispatched at 2026-08-19T07:21:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"e1eee6c793650a7c18d76cf2bccbd457734d3a9610df09f4e0a79b15d0c64711","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fec354a667895e675fed1f76acb6cd3afdc6bf9b42e6371dcffcf0807bbb4b53","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","STG"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T05:29:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000cdc8e8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-85gsqcc2zjlm\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/102c7ad9-cfd7-4fbb-b4af-47ae501c8b59\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/102c7ad9-cfd7-4fbb-b4af-47ae501c8b59\",\n      \"name\": \"102c7ad9-cfd7-4fbb-b4af-47ae501c8b59\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:41:39.807626461Z\",\n      \"endTime\": \"2026-08-19T08:05:45.761898963Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9geiu25jubct6f4b5k5ucmu6fmf5pl still exists (deletion dispatched at 2026-08-19T07:41:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000cdc8e8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-85gsqcc2zjlm\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/102c7ad9-cfd7-4fbb-b4af-47ae501c8b59\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/102c7ad9-cfd7-4fbb-b4af-47ae501c8b59\",\n      \"name\": \"102c7ad9-cfd7-4fbb-b4af-47ae501c8b59\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:41:39.807626461Z\",\n      \"endTime\": \"2026-08-19T08:05:45.761898963Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9geiu25jubct6f4b5k5ucmu6fmf5pl still exists (deletion dispatched at 2026-08-19T07:41:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"6aa10261ddbd41782888a9404c01284ae1dd4848412ef57729612cfd72e777cb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"8c42ccf15938c89725e1c4ff0149e014b02931be5a8b7de4535bfb0022695dd9","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T05:29:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000abefa8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-9b6mxq\" in resourcegroup=\"rg-np-noedge-9b6mxq-8gg2s6c8ddr4\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fc763824-a7c1-4c84-ad4a-52a04db780c3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fc763824-a7c1-4c84-ad4a-52a04db780c3\",\n      \"name\": \"fc763824-a7c1-4c84-ad4a-52a04db780c3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:11:47.482159856Z\",\n      \"endTime\": \"2026-08-19T07:35:54.369742847Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9g7tdod2lgoa5os2e9n6onp6vt2m2l still exists (deletion dispatched at 2026-08-19T07:11:48Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000abefa8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-9b6mxq\" in resourcegroup=\"rg-np-noedge-9b6mxq-8gg2s6c8ddr4\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fc763824-a7c1-4c84-ad4a-52a04db780c3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fc763824-a7c1-4c84-ad4a-52a04db780c3\",\n      \"name\": \"fc763824-a7c1-4c84-ad4a-52a04db780c3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:11:47.482159856Z\",\n      \"endTime\": \"2026-08-19T07:35:54.369742847Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9g7tdod2lgoa5os2e9n6onp6vt2m2l still exists (deletion dispatched at 2026-08-19T07:11:48Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"9c9601ec6e6a07ae3c3b84a7f7fd8991aebded251120b054ec5547f0853c5c70","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"220c303bbf477a74273991fde7c9a5f336a0663963edfed72cb63f983e8df885","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-21-140538 to 4.21.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T23:00:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-ltpqhc\" to 4.21.0-0.nightly-multi-2026-08-20-065923\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00071a3e0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-ltpqhc\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-c4ngqczk4ph5\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/80cf611d-a2f7-4a43-b5f4-cb99c877bca2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/80cf611d-a2f7-4a43-b5f4-cb99c877bca2\",\n      \"name\": \"80cf611d-a2f7-4a43-b5f4-cb99c877bca2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:42:45.54327206Z\",\n      \"endTime\": \"2026-08-21T23:42:47.666030908Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-21-140538 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-ltpqhc\" to 4.21.0-0.nightly-multi-2026-08-20-065923\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00071a3e0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-ltpqhc\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-c4ngqczk4ph5\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/80cf611d-a2f7-4a43-b5f4-cb99c877bca2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/80cf611d-a2f7-4a43-b5f4-cb99c877bca2\",\n      \"name\": \"80cf611d-a2f7-4a43-b5f4-cb99c877bca2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:42:45.54327206Z\",\n      \"endTime\": \"2026-08-21T23:42:47.666030908Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-21-140538 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","signature_id":"cae38e430d8e80e3a74035572e24eda7a12fa1071db2d9d61da01032369a656b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6357d57cb01557dc5021fba5497a64bf5694fab20b293c17f38284979b87902e","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-22-032601 to 4.21.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-22T23:00:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-ljjplf\" to 4.21.0-0.nightly-multi-2026-08-22-094635\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e00d00\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-ljjplf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-rjbgq4lgq7qt\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/14160601-762a-4228-a29b-548613a97ff8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/14160601-762a-4228-a29b-548613a97ff8\",\n      \"name\": \"14160601-762a-4228-a29b-548613a97ff8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:46:44.80771206Z\",\n      \"endTime\": \"2026-08-22T23:46:53.483788723Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-22-032601 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-ljjplf\" to 4.21.0-0.nightly-multi-2026-08-22-094635\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e00d00\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-ljjplf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-rjbgq4lgq7qt\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/14160601-762a-4228-a29b-548613a97ff8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/14160601-762a-4228-a29b-548613a97ff8\",\n      \"name\": \"14160601-762a-4228-a29b-548613a97ff8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:46:44.80771206Z\",\n      \"endTime\": \"2026-08-22T23:46:53.483788723Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-22-032601 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"f4249a53ee6a3bebd3274b03d3de9e11f0a0d607d4b225bdb1fc0bac2050d70c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"996c48750505877f54e5f4474f9ab414ba67007e2aa67aff8fc371e2a192e936","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-23-070610 to 4.21.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-wxm9hm\" to 4.21.0-0.nightly-multi-2026-08-22-094635\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a7c6c0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-wxm9hm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-m4n2k2rcnqcz\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ada4714-891c-4466-827a-ecc32dfa2442\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ada4714-891c-4466-827a-ecc32dfa2442\",\n      \"name\": \"1ada4714-891c-4466-827a-ecc32dfa2442\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T23:44:35.120120083Z\",\n      \"endTime\": \"2026-08-23T23:44:40.624916223Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-23-070610 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-wxm9hm\" to 4.21.0-0.nightly-multi-2026-08-22-094635\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a7c6c0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-wxm9hm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-m4n2k2rcnqcz\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ada4714-891c-4466-827a-ecc32dfa2442\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ada4714-891c-4466-827a-ecc32dfa2442\",\n      \"name\": \"1ada4714-891c-4466-827a-ecc32dfa2442\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T23:44:35.120120083Z\",\n      \"endTime\": \"2026-08-23T23:44:40.624916223Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-23-070610 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091661989947379712","occurred_at":"2026-08-23T23:00:54Z","signature_id":"0b2bd00bfed9399346e794c2683e5dec3d1647447257005ac477dccff872ac2c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f966d8a6f1dc892e4c6ee21571e53ba7d8d1ed39b375430e469f3616f54b9b95","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-24-135049 to 4.21.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-7z565t\" to 4.21.0-0.nightly-multi-2026-08-24-143120\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b5de0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-7z565t\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-98477z6czjf7\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n      \"name\": \"d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:45:23.595556181Z\",\n      \"endTime\": \"2026-08-24T23:45:30.682388649Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-24-135049 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-7z565t\" to 4.21.0-0.nightly-multi-2026-08-24-143120\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b5de0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-7z565t\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-98477z6czjf7\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n      \"name\": \"d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:45:23.595556181Z\",\n      \"endTime\": \"2026-08-24T23:45:30.682388649Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-24-135049 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"15bac440b3d4b1dd68415a557e13f0d806f6cdb816f9edcd9423e02924b01f2e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d8c04d75e2d056f1b8b36668a13ab7045b056c9f740c7013c4882cfcfef5d2d6","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.21.0-0.nightly-multi-2026-08-24-143120 to 4.22.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-cbj2dm\" to 4.22.0-0.nightly-multi-2026-08-23-080550\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000df2720\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-cbj2dm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4ls4ksxqchgg\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n      \"name\": \"d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:48:14.416516082Z\",\n      \"endTime\": \"2026-08-24T23:48:20.787265431Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.21.0-0.nightly-multi-2026-08-24-143120 to 4.22.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-cbj2dm\" to 4.22.0-0.nightly-multi-2026-08-23-080550\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000df2720\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-cbj2dm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4ls4ksxqchgg\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n      \"name\": \"d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:48:14.416516082Z\",\n      \"endTime\": \"2026-08-24T23:48:20.787265431Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.21.0-0.nightly-multi-2026-08-24-143120 to 4.22.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"498a13bc157b497dedc0c407f7c032fb09b28d4110c6fbee95fdae33ff5fe349","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"8de5e14024f45fb18338f1663f4f58e82e3a8afdfe682bc869aa5a0214bf4fc2","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-17-010411 to 4.23.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T23:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:174]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-d5pz9g\" to 4.23.0-0.nightly-multi-2026-08-19-140455\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000900d00\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-d5pz9g\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-ttgbbn52wj5b\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f80b855b-6ddb-4d41-bcfd-69a629fa5f3a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f80b855b-6ddb-4d41-bcfd-69a629fa5f3a\",\n      \"name\": \"f80b855b-6ddb-4d41-bcfd-69a629fa5f3a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:46:03.25868454Z\",\n      \"endTime\": \"2026-08-19T23:46:13.75881135Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-17-010411 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:174]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-d5pz9g\" to 4.23.0-0.nightly-multi-2026-08-19-140455\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000900d00\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-d5pz9g\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-ttgbbn52wj5b\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f80b855b-6ddb-4d41-bcfd-69a629fa5f3a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f80b855b-6ddb-4d41-bcfd-69a629fa5f3a\",\n      \"name\": \"f80b855b-6ddb-4d41-bcfd-69a629fa5f3a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:46:03.25868454Z\",\n      \"endTime\": \"2026-08-19T23:46:13.75881135Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-17-010411 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090212357799153664","occurred_at":"2026-08-19T23:00:07Z","signature_id":"6e67a0588318a98d023efbba4cd0e38a13983e93b7d4dccafc848765ed882535","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"93eb9aed3c673a25dda3025e9f38f8d1c53f9c491622feaa90552b1afbad3297","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-20-141954 to 4.23.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:174]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-r7msf7\" to 4.23.0-0.nightly-multi-2026-08-20-085020\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000db4400\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-r7msf7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-cxr6vwskbzqs\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cbae144c-9ad6-4399-8e1e-708ba5343570\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cbae144c-9ad6-4399-8e1e-708ba5343570\",\n      \"name\": \"cbae144c-9ad6-4399-8e1e-708ba5343570\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T23:45:35.456682564Z\",\n      \"endTime\": \"2026-08-20T23:45:36.715219027Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-20-141954 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:174]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-r7msf7\" to 4.23.0-0.nightly-multi-2026-08-20-085020\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000db4400\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-r7msf7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-cxr6vwskbzqs\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cbae144c-9ad6-4399-8e1e-708ba5343570\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cbae144c-9ad6-4399-8e1e-708ba5343570\",\n      \"name\": \"cbae144c-9ad6-4399-8e1e-708ba5343570\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T23:45:35.456682564Z\",\n      \"endTime\": \"2026-08-20T23:45:36.715219027Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-20-141954 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090574634771550208","occurred_at":"2026-08-20T23:00:08Z","signature_id":"a5317b67ffb05362e28d05bcc9279b1afb1324ef6112b96fcdc34cfd4192cc2e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f9bfb1faf51b51475c30265bfb24f836c9386a47bad2413a3d8e330ed7c9045e","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-20-215447 to 4.23.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T23:00:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-7jwrlb\" to 4.23.0-0.nightly-multi-2026-08-21-150158\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00071fec0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-7jwrlb\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-ddtbb45g74fw\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\",\n      \"name\": \"47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:45:19.683338639Z\",\n      \"endTime\": \"2026-08-21T23:45:20.631024642Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-20-215447 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-7jwrlb\" to 4.23.0-0.nightly-multi-2026-08-21-150158\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00071fec0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-7jwrlb\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-ddtbb45g74fw\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\",\n      \"name\": \"47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:45:19.683338639Z\",\n      \"endTime\": \"2026-08-21T23:45:20.631024642Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-20-215447 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","signature_id":"6ce3f21db4e03dbc02da614b27bd974fe76c85c6dcbc700f959708a7547d5026","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9ae716254ca5ddb2b82f158d6a7b9ae7aabc6066c7efeaa8525d2090a51a812e","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-21-221014 to 4.23.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-22T23:00:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-z5pqzx\" to 4.23.0-0.nightly-multi-2026-08-22-063717\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012ac660\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-z5pqzx\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-227n7tlwrhnd\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ef2c5e8e-3ad7-4179-842e-7126874115be\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ef2c5e8e-3ad7-4179-842e-7126874115be\",\n      \"name\": \"ef2c5e8e-3ad7-4179-842e-7126874115be\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:47:18.723854025Z\",\n      \"endTime\": \"2026-08-22T23:47:19.754634857Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-21-221014 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-z5pqzx\" to 4.23.0-0.nightly-multi-2026-08-22-063717\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012ac660\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-z5pqzx\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-227n7tlwrhnd\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ef2c5e8e-3ad7-4179-842e-7126874115be\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ef2c5e8e-3ad7-4179-842e-7126874115be\",\n      \"name\": \"ef2c5e8e-3ad7-4179-842e-7126874115be\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:47:18.723854025Z\",\n      \"endTime\": \"2026-08-22T23:47:19.754634857Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-21-221014 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"6b3d20a396a74d44cdd8a7cc9268f6dc244330e48dd00a260ef2f088a769b44f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3b509588f678a0fd976eb453004a3e8a55c0be5c12f8340c463453a657ce6678","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-24-152026 to 5.1.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-q6hwb9\" to 5.1.0-0.nightly-multi-2026-08-24-111752\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a25da0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-q6hwb9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-kgxwzfcjcc2b\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n      \"name\": \"7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:51:24.1449981Z\",\n      \"endTime\": \"2026-08-24T23:51:30.86207539Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-24-152026 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-q6hwb9\" to 5.1.0-0.nightly-multi-2026-08-24-111752\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a25da0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-q6hwb9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-kgxwzfcjcc2b\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n      \"name\": \"7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:51:24.1449981Z\",\n      \"endTime\": \"2026-08-24T23:51:30.86207539Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-24-152026 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"05526aa86d5e4dd680e08fb3cf223532281a30bb281575b5c8abfec129525d60","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":17,"also_in":["DEV","STG"],"trend_counts":[1,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":23,"prior_last_seen_at":"2026-08-19T08:22:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"925c581adc4f4ee74421c479e1f09406ff5021fa3c71cc253a16495b01df4b7d","failure_pattern":"failed to update cluster \"\u003ccluster\u003e\" authorized CIDRs to include test runner IP","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-22T23:00:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:249]: failed to update cluster \"agg-cluster\" authorized CIDRs to include test runner IP\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:249]: failed to update cluster \"agg-cluster\" authorized CIDRs to include test runner IP\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"da4948d61ab54e112557ad6fe48bbeb3135dc01594a49200da12544749a36fa1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":6,"also_in":["INT","STG"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":12,"prior_last_seen_at":"2026-08-19T02:00:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000d24198\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-xjt5s.apps.aro.cilium-cl.j5nd.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000d24198\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-xjt5s.apps.aro.cilium-cl.j5nd.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"6985a3af1e1b7d9f14fa3e6ebca829c6de310ae27552aab65799f5ab8b2a33dd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"7137d3f115bf4c264bb74c888bdc6c3954a56a37032931fb9aa8d47ff31a4953","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.857142857142857,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T10:34:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.002s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.002s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2090024653178277888","occurred_at":"2026-08-19T10:34:42Z","signature_id":"85944d4fa3cc8f9a02514637d6119a2fd69da50caa8d14fdae25e1e32d2d5f52","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":24,"failed_runs":21,"raw_occurrences":204,"matched_occurrences":203,"runs_affected":20},"rows":[{"environment":"stg","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":55,"runs_affected":8,"failed_runs":8,"run_impact_percent":33.333333333333336,"anchor_occurrences":55,"after_last_push_count":55,"also_in":["INT","PROD"],"trend_counts":[0,0,2,10,17,18,8,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":9,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001990018\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-58vq7.apps.aro.cilium-cluster.xjbu.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001990018\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-58vq7.apps.aro.cilium-cluster.xjbu.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0023089a8\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-gd2m6.apps.aro.agg-cluster.sh9x.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0023089a8\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-gd2m6.apps.aro.agg-cluster.sh9x.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc002bc60a8\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-2xkbq.apps.aro.cilium-cl.yqg3.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc002bc60a8\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-2xkbq.apps.aro.cilium-cl.yqg3.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"128bdc5bd9fd4d676037c7b3876358423100506abaac1cb7c6a2aa6dca99f9d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"268af4cfefc9d9ce5477c6309e34d9e625934c5fec5612737f144e143459ca23","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"568397239d373e84cc1193077022f2929fdbc7536d41e0a023cda085a91923ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"64d7370897c1ddb0a4c082a98b5de0abd894ee5bd836383e651a1f53f91a033d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"6f400eea163c9e3d28c8b1f64accc1cf5963abf78c260413f5fe388a60f0006c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"aa4dc686c4f2ffb192f550321a5d618c671a84350ba22f26b4ee1aaac5686fb0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"c217516f086c4ee518f66b5fc0b63d7e19723573004fb051ee9ac9e2e83e0fb9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"de74f8de034a6e8da10a240f9e0d6f03ee3be2522666aaf0b6ad709ddf3efbc4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"0652b92117d22a5b2e9cf20aa5637d5c27db260e7c8ad186d851762e8f90ade7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"0c968f59cb717431d5c7d3b774ddd58e54f3c9e98d2283d5ed8c612c784ab28f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"22b5ae279ed4a4d36e707040d191eba08b365c5589624a6177f37632ecb6dd27","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"4e439067d1f887fd6586941fe0a591f509c53905e7b1e54cc808ee63599dd229","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"56a2d316ec317e48edc96d53b9c004ddce28b2b100d6c2fbc7d62acc3230859f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"81e44a56cd141302f440c9207b5cd0d22a87018ab48353bc76f47efbe9dc9da6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"8f6da9f6c6a1f82e13e70219a05bdb624dbe89ba1d87c0b4d0e1e9ad11d1b63d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"917a7ab84fc8483a4bd3b2ce2620cefe932a4a57e8830c6d4a4588716e952577","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"a9220eabfed81c9d6a05b1347966522bde35e0e1de1e1d5420f8bfe1ce00e1b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"dff5d87b8dd7ae53a959d7abd4ee8e7071329d518e20c863334623937ac665a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"099c7f9ec6546f82588b173d8d978935aefdaf7c422c94012f4b5bcff2a2bd2f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"358e627c5a3717e340077769b0bd08ea4a344ecd196320db476ee3f70bc37bd4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"3b1f4a9168c319727c1d0cdb76f21a361aff2705a016f6e2bd6ce401fdc245a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"4b0b2bdfb319f0241a615594f15b72f98d287b016595aa560833ca643e7c41b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"4e525635a334f496f86e5447f7bfcc41a3027f81d279cac4a2c98a911e1a14e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"b0305dfd29e4255374c19ac1807f6ec0adeadfa5faf330d938857c76cee28385","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"ca5231bfe441f0896ee14844376689aae71fdb84bd0ceb16d221be71b2d1f758","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"eef5d362c6ca5ec89cb57e6a06e53474bbe066166bdea8cc01dfe1eee398b32d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"11a725162d782d59307e54b01812f83276c8b768291c214985b0e7cf30d62295","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"239e965421538be1363dd2c7f07fdf5811ab39dbc83ef7b4d067112bb5d9f0fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"3cd272fe93bf03fdbe630ac01847905ab2758c75271e802033f6c152ac24510d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"473ae6affbd1614b127350a684f5c57cf48befc5b71ca9167deb53482d16e04e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"5aea9a96af15785f8a001a629201e85cfd8f31b12c83f95c9952b9431240bf9b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"5b0f787f537c20118ee76f9fc63fd06eb58ee0eee7e5ee4575316f4340472c37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"7ffc802cb180f4209a0fa858155687cc6e2899d3764fe7d6bd1321ae4fd55e84","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"8c6b27faba09bc535bc427b19d9416db95aeff2cd1af3b110ae29985fbe189dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"c5fb4bff03f86320f1edc3eea596547758a86968f42b91dffea826dc780ab8c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"fdf84045828cfe2df5b8ebc4b5d8480c96306c6e7327c688db9c24bc34502f64","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"15cc422b2f3be8fc9e52950819b6911fd45c6c141421a6134c4d451278306bca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"1b080e7bcd141068d87382a1bcd086ccda9b6dd57cad2be71077c408d2e7e438","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"7d8117c22146bf5a0774f739c19b87fd0cbd57d6a02884d31d96768d87ca983d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"907dec55a9074061a33272193d894502c57d11b65a9f5f49e62962d638dd0ba1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"9a9e1cb7ce25683e8a9ba03d52e031fa731f92999b78652a943b0a33011dc620","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"db6298b2d5a06ac0c0b6891a012777daa148bd9f9827d3298b5f01d7948507b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"f0207e912b5808bacb31a7fe73aaf94e3c55660884e013054934fb98f7a335f8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"014faee8c65110da8bf653c059bda71ae8ef3480a383a2999fc04af379294b99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"0e428625a39042ad70067f1e3a5ffb3ef2deec8808b6df01f291fa0854988be4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"115305b069b0aa8d64167995814d5cd6d32a1f792870e701a3affd4a8b96bdfe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"587e72c577de17291ef8b6b3ac4f6b12a0663bf2c56e0f53f30373aec8471091","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"6073bf034d1aabb50f60a9559998d196f90b404e7a6f2c7c9d69ac4bbe1020b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"697a4c129533a315c9016b84da122b5bc17c7516225d6ed78343e011c45dd4ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"69ff046ff55d8bfc7023c374da3602741135d47054bcf695f149cb3279b1d92b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"a1166de0889c8bcb931d1a2474d9ac58b2df20920fe3d4ae0d4b27104be80122","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"a610245db2dc06bccba732a90d5f0ee6d5e2651fd4bb3d357ae0d19e0833cec8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"bf0b84a697d66c570dffb08e752d5c50734534e54f512951dc11b3757ac8cd8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"4649980ccb572a8df4fef4e44831f1de4eae2d176d8ecffa92fb99f1ae87aa7d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"bf90b8b00390291912ae577c4eaa041b4552832f8c625cf2cd2cff628c244b5a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":19,"runs_affected":5,"failed_runs":5,"run_impact_percent":20.833333333333332,"anchor_occurrences":19,"after_last_push_count":19,"also_in":["DEV","INT","PROD"],"trend_counts":[0,3,0,4,4,4,4,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0012c61b0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-23-080550, targetMinor=4.23.0-0.nightly-multi-2026-08-24-110923) failed: clusterversion status.history has no version in target minor \"4.23.0-0.nightly-multi-2026-08-24-110923\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0012c61b0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-23-080550, targetMinor=4.23.0-0.nightly-multi-2026-08-24-110923) failed: clusterversion status.history has no version in target minor \"4.23.0-0.nightly-multi-2026-08-24-110923\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000013c68\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-24-135049, targetMinor=4.21.0-0.nightly-multi-2026-08-24-143120) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-24-143120\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000013c68\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-24-135049, targetMinor=4.21.0-0.nightly-multi-2026-08-24-143120) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-24-143120\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001423c38\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-24-143120, targetMinor=4.22.0-0.nightly-multi-2026-08-23-080550) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-23-080550\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001423c38\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-24-143120, targetMinor=4.22.0-0.nightly-multi-2026-08-23-080550) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-23-080550\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"1f961d63e5efbc9f674297ab6ff360e18191616cad68297a32bfc0352b1efd45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"5d51d6400541f1ef2ea735bc34f88c2cca7f1b91d8f8680efed7dfe0ab6e075d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"8c228d79217e5703c43c091d1bbcb6c74c0d2e48dd70a8ed3bbc113b8600d236","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"9a6af90baac0bf0367bb530e542307559248b48af09dc4fa5550242ec09f64e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"4130f29b010aeb34f9c8f43a05c2e44d7785c514f2add915cf23a436d9b27ff7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"7a1d942e87a8503eedbb424b862f5ab7d117d1ff924fa3f4d4d4d997e3166d93","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"9cf68146343a69d8617ea02654d3caa9f50c984ec8e49043fd90de1d043d4cef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"e8776a142c237294cf872e39df27cf70f10aa1c9176e12b51594515a2d850deb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"444e0b46276d6b06038a0349a8a8b1b9c6bd0d97e65d53c08b2f7737e27ce63b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"6183ddd6caf2d7afde8bcd7be49b7fe37da55024a657dec1d775306e9477f62a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"87802490969f4a6d1eb8eb8df9ca6f5f32bcb5b82112d8d6b1e304b1d96e3e03","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"b7e9a22c6cf3ac389f2a77fd9b2625cddcf6fd44e337c61d636e8df2c2743c61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"62bbc0ad65ca651617994b83d05a4a4fe629d762c5e177de75e71c5449cc78fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"86e1eb29ec597937a663760329209bb50694b5af2a87b7c19ee3c634d1801423","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"b880458c73aad38c09b6646920150ac274a35d4170590ae9673504a08162d18b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"e6a8ea22f064439192de7c94bd0b019d27fcf10b53a96a3dbbeaabdae0a1680a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"1d432859b9b61eadeae3f59e9810dbfd5dbaf0c705a7bb7f6f36b66b56115528","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"2695f4fff945203d2d453601faacf71292783cffa1d1c099df7d765bed100038","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"b5edad6c45e84113c6ceffd16aae87af35a9ee427b9bd298e22a3b075a317a70","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":10,"runs_affected":8,"failed_runs":8,"run_impact_percent":33.333333333333336,"anchor_occurrences":10,"after_last_push_count":10,"also_in":["DEV","PROD"],"trend_counts":[0,4,4,2,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-21T23:00:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011964b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-h8fb74\" in resourcegroup=\"rg-np-version-upgrade-h8fb74-s5mqnw56gx4w\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/701376c3-ab04-47a8-a1ef-8375724f8f86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/701376c3-ab04-47a8-a1ef-8375724f8f86\",\n      \"name\": \"701376c3-ab04-47a8-a1ef-8375724f8f86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T00:26:59.095965518Z\",\n      \"endTime\": \"2026-08-22T00:51:01.463363445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb99fkh6uh3oo5nr4qni7jhed5o0tgt still exists (deletion dispatched at 2026-08-22T00:26:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011964b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-h8fb74\" in resourcegroup=\"rg-np-version-upgrade-h8fb74-s5mqnw56gx4w\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/701376c3-ab04-47a8-a1ef-8375724f8f86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/701376c3-ab04-47a8-a1ef-8375724f8f86\",\n      \"name\": \"701376c3-ab04-47a8-a1ef-8375724f8f86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T00:26:59.095965518Z\",\n      \"endTime\": \"2026-08-22T00:51:01.463363445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb99fkh6uh3oo5nr4qni7jhed5o0tgt still exists (deletion dispatched at 2026-08-22T00:26:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013c8138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-2p9lgg975wsf\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5710f93-c135-4ecd-84ba-bd766a4f3157\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5710f93-c135-4ecd-84ba-bd766a4f3157\",\n      \"name\": \"d5710f93-c135-4ecd-84ba-bd766a4f3157\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:51:19.851300811Z\",\n      \"endTime\": \"2026-08-21T03:15:24.947768689Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2samnf949m9mks8utf8hvjl3i9ocmvd0 still exists (deletion dispatched at 2026-08-21T02:51:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013c8138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-2p9lgg975wsf\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5710f93-c135-4ecd-84ba-bd766a4f3157\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5710f93-c135-4ecd-84ba-bd766a4f3157\",\n      \"name\": \"d5710f93-c135-4ecd-84ba-bd766a4f3157\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:51:19.851300811Z\",\n      \"endTime\": \"2026-08-21T03:15:24.947768689Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2samnf949m9mks8utf8hvjl3i9ocmvd0 still exists (deletion dispatched at 2026-08-21T02:51:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011f24f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listwvkfxb\" in resourcegroup=\"ea-list-q8zktqx9k8tx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e8f24bc-6934-4503-b388-7fea73015050\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e8f24bc-6934-4503-b388-7fea73015050\",\n      \"name\": \"7e8f24bc-6934-4503-b388-7fea73015050\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T14:04:49.548371908Z\",\n      \"endTime\": \"2026-08-20T14:28:52.367967433Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sabeurund3r70lf12nj56m098p5j7j9 still exists (deletion dispatched at 2026-08-20T14:04:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011f24f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listwvkfxb\" in resourcegroup=\"ea-list-q8zktqx9k8tx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e8f24bc-6934-4503-b388-7fea73015050\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e8f24bc-6934-4503-b388-7fea73015050\",\n      \"name\": \"7e8f24bc-6934-4503-b388-7fea73015050\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T14:04:49.548371908Z\",\n      \"endTime\": \"2026-08-20T14:28:52.367967433Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sabeurund3r70lf12nj56m098p5j7j9 still exists (deletion dispatched at 2026-08-20T14:04:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"ca75b2343ff23a11e0fce1a31c9ef24459c4c103d790af5c0d8320fc327b3727","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"afbd9ea61b7f5f2cf2d4e5eb07c2fecfd15d1778af4a847a49ff12ce110071ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"c98eeb6d2064c80a5c88dac50a9f84605cf8ee1eb70a70345435f7284fcc94c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"0facf4aca2cbbc09d1d4d2b5f2a2c443dc9ac7bce839a50cfc50190997e06fd4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"a1399ba06ca55ef9ac4602a99872b2b1d13798c7ec52bf26477719dd69c14e17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","signature_id":"d19ec4d4f31d760d02b50b389f562a65f23856356a3f633669a10c48565d949d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090159303129305088","occurred_at":"2026-08-19T19:29:45Z","signature_id":"e3d4b835e521fff377b185d308d1607a3e78fb8712e5a6bf8fcb1ba092f6e484","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090159303129305088","occurred_at":"2026-08-19T19:29:45Z","signature_id":"e4f8b47f31271f0cf76b5df5def7ef54ddf1770bbdfd048bee2fd57247eec8de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090064577516539904","occurred_at":"2026-08-19T13:13:21Z","signature_id":"c952a9e95cca1027c298ae780c02d302a431ed86a3d72fe39ce0c575ad12c319","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089951420068401152","occurred_at":"2026-08-19T05:43:42Z","signature_id":"0f4eaab480b1c780fe060d154f73a8d802037c158a76021e9a8d565c30c9bbe5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"819462e5c945d35c3bda97a6aae4ae2bf1db2fba1cd848aee697a026ed6e2265","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":9,"runs_affected":3,"failed_runs":3,"run_impact_percent":12.5,"anchor_occurrences":9,"after_last_push_count":9,"also_in":["DEV"],"trend_counts":[0,0,3,6,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:140]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-xbp8kt\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d57060\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-xbp8kt: failed waiting for cluster=\"cp-ystream-upgrade-4-21-xbp8kt\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-mvbldsj9x64s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0b50fbb-5e08-405c-bc27-320fd18ae286\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0b50fbb-5e08-405c-bc27-320fd18ae286\",\n      \"name\": \"d0b50fbb-5e08-405c-bc27-320fd18ae286\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:33:00.457364861Z\",\n      \"endTime\": \"2026-08-21T02:52:03.680608781Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, control-plane-pki-operator, kube-scheduler, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, ignition-server, olm-operator, machine-approver, olm-collect-profiles, openshift-apiserver, cluster-policy-controller, csi-snapshot-controller-operator, catalog-operator, packageserver, community-operators-catalog, cluster-autoscaler, cluster-version-operator, certified-operators-catalog, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, dns-operator, konnectivity-agent, redhat-operators-catalog, kube-controller-manager, openshift-controller-manager, hosted-cluster-config-operator, ignition-server-proxy, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:140]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-xbp8kt\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d57060\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-xbp8kt: failed waiting for cluster=\"cp-ystream-upgrade-4-21-xbp8kt\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-mvbldsj9x64s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0b50fbb-5e08-405c-bc27-320fd18ae286\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0b50fbb-5e08-405c-bc27-320fd18ae286\",\n      \"name\": \"d0b50fbb-5e08-405c-bc27-320fd18ae286\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:33:00.457364861Z\",\n      \"endTime\": \"2026-08-21T02:52:03.680608781Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, control-plane-pki-operator, kube-scheduler, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, ignition-server, olm-operator, machine-approver, olm-collect-profiles, openshift-apiserver, cluster-policy-controller, csi-snapshot-controller-operator, catalog-operator, packageserver, community-operators-catalog, cluster-autoscaler, cluster-version-operator, certified-operators-catalog, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, dns-operator, konnectivity-agent, redhat-operators-catalog, kube-controller-manager, openshift-controller-manager, hosted-cluster-config-operator, ignition-server-proxy, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f0f080\u003e: \n    failed to create HCP cluster np-downgrade-kjw6pt: failed waiting for cluster=\"np-downgrade-kjw6pt\" in resourcegroup=\"rg-np-downgrade-kjw6pt-lf69f9frnjjp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98f96261-7c0c-4604-82d2-ccf83d040db2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98f96261-7c0c-4604-82d2-ccf83d040db2\",\n      \"name\": \"98f96261-7c0c-4604-82d2-ccf83d040db2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:48:21.62730862Z\",\n      \"endTime\": \"2026-08-21T03:07:24.73168706Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-storage-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-route-controller-manager, cluster-policy-controller, ingress-operator, cluster-image-registry-operator, certified-operators-catalog, konnectivity-agent, catalog-operator, redhat-operators-catalog, control-plane-pki-operator, hosted-cluster-config-operator, packageserver, community-operators-catalog, kube-scheduler, azure-cloud-controller-manager, openshift-apiserver, cluster-version-operator, dns-operator, olm-operator, olm-collect-profiles, kube-controller-manager, cluster-network-operator, redhat-marketplace-catalog, cluster-autoscaler, machine-approver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f0f080\u003e: \n    failed to create HCP cluster np-downgrade-kjw6pt: failed waiting for cluster=\"np-downgrade-kjw6pt\" in resourcegroup=\"rg-np-downgrade-kjw6pt-lf69f9frnjjp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98f96261-7c0c-4604-82d2-ccf83d040db2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98f96261-7c0c-4604-82d2-ccf83d040db2\",\n      \"name\": \"98f96261-7c0c-4604-82d2-ccf83d040db2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:48:21.62730862Z\",\n      \"endTime\": \"2026-08-21T03:07:24.73168706Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-storage-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-route-controller-manager, cluster-policy-controller, ingress-operator, cluster-image-registry-operator, certified-operators-catalog, konnectivity-agent, catalog-operator, redhat-operators-catalog, control-plane-pki-operator, hosted-cluster-config-operator, packageserver, community-operators-catalog, kube-scheduler, azure-cloud-controller-manager, openshift-apiserver, cluster-version-operator, dns-operator, olm-operator, olm-collect-profiles, kube-controller-manager, cluster-network-operator, redhat-marketplace-catalog, cluster-autoscaler, machine-approver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbc300\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-dnkv5vg5t2ts\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/abfca9a0-da31-4fda-b740-f92e856a7dca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/abfca9a0-da31-4fda-b740-f92e856a7dca\",\n      \"name\": \"abfca9a0-da31-4fda-b740-f92e856a7dca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:36.161779463Z\",\n      \"endTime\": \"2026-08-21T02:22:42.321981114Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-node-tuning-operator, csi-snapshot-controller-operator, ignition-server-proxy, cluster-policy-controller, azure-cloud-controller-manager, cluster-storage-operator, konnectivity-agent, olm-collect-profiles, redhat-marketplace-catalog, certified-operators-catalog, packageserver, community-operators-catalog, machine-approver, control-plane-pki-operator, ignition-server, openshift-controller-manager, dns-operator, cluster-autoscaler, kube-controller-manager, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-network-operator, cluster-version-operator, ingress-operator, cluster-image-registry-operator, catalog-operator, olm-operator, openshift-apiserver, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbc300\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-dnkv5vg5t2ts\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/abfca9a0-da31-4fda-b740-f92e856a7dca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/abfca9a0-da31-4fda-b740-f92e856a7dca\",\n      \"name\": \"abfca9a0-da31-4fda-b740-f92e856a7dca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:36.161779463Z\",\n      \"endTime\": \"2026-08-21T02:22:42.321981114Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-node-tuning-operator, csi-snapshot-controller-operator, ignition-server-proxy, cluster-policy-controller, azure-cloud-controller-manager, cluster-storage-operator, konnectivity-agent, olm-collect-profiles, redhat-marketplace-catalog, certified-operators-catalog, packageserver, community-operators-catalog, machine-approver, control-plane-pki-operator, ignition-server, openshift-controller-manager, dns-operator, cluster-autoscaler, kube-controller-manager, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-network-operator, cluster-version-operator, ingress-operator, cluster-image-registry-operator, catalog-operator, olm-operator, openshift-apiserver, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"0ed7bae60681c60f68c493d90cb1fd137349d28e37ec331f94e67f0f0671c10a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"238e7b24b648c5d6d6127862df062515aa947e0f889da3e18d7d7c8417d0316a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"32a0bd18972a8f8e83478af297e9368419e1b7e7597a308653070e30fe4fe6c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"37d9e7ef301047c2b2fd4899c663861930478c9a873b27090cbd860df00a8a91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"950772107db22c8db323365cbb8dc7ad54495494533bdb154fc48dbef42f844b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"eaae9ffa124a963d7a6edae9a93b0de32ff8eeb2a17d21c5b6b88689e4aad9b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"fd1a222e88fac51899884139c985b18a14bda354bbdc6d03b6bd0f1ec3b9916a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"fd7348293825c9d41446d338b7d31ffd23035dfeb87dc5ea60ab54b00dd1a82a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"f86b3413a4fa89f0fe3aee0a97c3b3aae586cbd24a62a162177322eec5c926c9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-nrmbvd, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: failed waiting for cluster=\"cluster-candidate-5-0-nrmbvd\" in resourcegroup=\"rg-candidate-5-0-dgkn5p9l9flc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":8,"runs_affected":7,"failed_runs":7,"run_impact_percent":29.166666666666668,"anchor_occurrences":8,"after_last_push_count":8,"also_in":["DEV","PROD"],"trend_counts":[0,2,2,1,1,1,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":26,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":6}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002ac4480\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-gp6bzmxwxfps\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002ac4480\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-gp6bzmxwxfps\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001a9cb70\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-4mnnfzb2nhtb, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-4mnnfzb2nhtb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-4mnnfzb2nhtb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001a9cb70\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-4mnnfzb2nhtb, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-4mnnfzb2nhtb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-4mnnfzb2nhtb, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002b9e330\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fg554p8tjmmf, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-fg554p8tjmmf\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fg554p8tjmmf, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002b9e330\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fg554p8tjmmf, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-fg554p8tjmmf\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fg554p8tjmmf, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"1ecfdcaa8fd2583c7f65498931069ae0e42fa1b933a7ed7714a974ea82a7b373","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"5554100c5cb6fda2b51ccf73571621c9937fb5b9897851e954053e5c8912e0a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"4f5c940afced23891d414938cb4cb38d59030d0411748cce89554393d71b5e2c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"9d8ff92f48ab455bb4637f037f6fb65f0f9eab5e788e3363f3d8f0a842a4a77d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"5b48ea80fc398a034dba3d46b19e592386ab5e5708b0a0e23267c0282b5a55be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","signature_id":"2c025bacef2a95219db5cc2c1d0f8e25156096d570a86e526e43447bf8f9a29b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"6f01fa48d2614fdfbfe9c406cf323639e0a1db4176d42dd1853204a6a7a20910","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"ba0f03a868ed9f5c8c401dbf4f120f2223529a931c2927647bf01ebba2dbf3c4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"acf4c4a2d46e5ea4b872812ba9f4a18d449fa2e31dbb376d0d126db4afe5c289","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":8,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":8,"after_last_push_count":8,"trend_counts":[0,0,0,8,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cc9300\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-6cppfxw8g4qh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\",\n      \"name\": \"1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:28.697047474Z\",\n      \"endTime\": \"2026-08-21T02:22:32.526910358Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cc9300\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-6cppfxw8g4qh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\",\n      \"name\": \"1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:28.697047474Z\",\n      \"endTime\": \"2026-08-21T02:22:32.526910358Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f84520\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-fq28q2k4mqnr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\",\n      \"name\": \"7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:19.705086647Z\",\n      \"endTime\": \"2026-08-21T02:22:22.572455462Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f84520\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-fq28q2k4mqnr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\",\n      \"name\": \"7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:19.705086647Z\",\n      \"endTime\": \"2026-08-21T02:22:22.572455462Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-wlltck9mnpdh/cluster-candidate-4-21-wqbh9c should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e0c460\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-wqbh9c: failed waiting for cluster=\"cluster-candidate-4-21-wqbh9c\" in resourcegroup=\"rg-candidate-4-21-wlltck9mnpdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/61fe2608-61d1-400b-90c7-806bb5489f2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/61fe2608-61d1-400b-90c7-806bb5489f2b\",\n      \"name\": \"61fe2608-61d1-400b-90c7-806bb5489f2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:28.583859524Z\",\n      \"endTime\": \"2026-08-21T02:22:32.536144456Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-wlltck9mnpdh/cluster-candidate-4-21-wqbh9c should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e0c460\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-wqbh9c: failed waiting for cluster=\"cluster-candidate-4-21-wqbh9c\" in resourcegroup=\"rg-candidate-4-21-wlltck9mnpdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/61fe2608-61d1-400b-90c7-806bb5489f2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/61fe2608-61d1-400b-90c7-806bb5489f2b\",\n      \"name\": \"61fe2608-61d1-400b-90c7-806bb5489f2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:28.583859524Z\",\n      \"endTime\": \"2026-08-21T02:22:32.536144456Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"0c797b790f34212a0962c3e54028dc04c84df9396c80443411635e702cab7abf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"43f28e097ff6599e427ad726f1fa1c75940fb0b825513844f3957b76cf6b817b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"565fce7fcb4445a79f30034d53708e522a56029627f0a6e682c9320663ffa91f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"6a04db7e50f70ea9dac9932acea60a13f6c9b7d4f8617adb14a39552f096a365","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"81fc71e1b2add474a2f9142629a0d6332c8b66a31ce3ed781eff6cdd129c1fb1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"82fc9dfcd4a1d1dc958933049664f71eaf4c60ebd9aac5ce72d39c5595e18a42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"b84e75c4953bfe9005b85e725e144df6a003f1fa4344d9a74c269a1620ea8307","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"dc0cc1f46b4c3a94844aceabaa9ad9108ada785e52f771e817b2071398fc7bb1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3b73b158de2cfa00931d006d7dea813139c9ea99ca292b8ded3d10d4beb4674f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":8,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":8,"after_last_push_count":9,"also_in":["DEV","INT","PROD"],"trend_counts":[0,0,0,8,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001a80bd0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0a6ef0ec-688d-48ac-96fb-22ede87538f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0a6ef0ec-688d-48ac-96fb-22ede87538f5\",\n      \"name\": \"0a6ef0ec-688d-48ac-96fb-22ede87538f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:04:28.992322331Z\",\n      \"endTime\": \"2026-08-21T02:23:32.540875768Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001a80bd0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0a6ef0ec-688d-48ac-96fb-22ede87538f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0a6ef0ec-688d-48ac-96fb-22ede87538f5\",\n      \"name\": \"0a6ef0ec-688d-48ac-96fb-22ede87538f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:04:28.992322331Z\",\n      \"endTime\": \"2026-08-21T02:23:32.540875768Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cc1940\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-p8kxn92zjqhp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5185baff-cb0e-4088-baa6-c6571cadd51f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5185baff-cb0e-4088-baa6-c6571cadd51f\",\n      \"name\": \"5185baff-cb0e-4088-baa6-c6571cadd51f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:36.75245159Z\",\n      \"endTime\": \"2026-08-21T02:22:42.448207861Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cc1940\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-p8kxn92zjqhp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5185baff-cb0e-4088-baa6-c6571cadd51f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5185baff-cb0e-4088-baa6-c6571cadd51f\",\n      \"name\": \"5185baff-cb0e-4088-baa6-c6571cadd51f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:36.75245159Z\",\n      \"endTime\": \"2026-08-21T02:22:42.448207861Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000edcea0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k8fl8dx2bw22\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/42d6143a-e97b-4483-afd9-f74bce1b7086\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/42d6143a-e97b-4483-afd9-f74bce1b7086\",\n      \"name\": \"42d6143a-e97b-4483-afd9-f74bce1b7086\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:44:46.647452675Z\",\n      \"endTime\": \"2026-08-21T03:03:54.737093461Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000edcea0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k8fl8dx2bw22\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/42d6143a-e97b-4483-afd9-f74bce1b7086\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/42d6143a-e97b-4483-afd9-f74bce1b7086\",\n      \"name\": \"42d6143a-e97b-4483-afd9-f74bce1b7086\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:44:46.647452675Z\",\n      \"endTime\": \"2026-08-21T03:03:54.737093461Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"001ed7fa69d2a092f2b072655683cfb30462642d1bd846ab3b9b253dbcf3dab5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"0027226a133920ef1c26470ce81c33d48a4b562da304b31b0fe5ea881d2172b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"684f37cf48d101c235cdeb5cebaf623acfb01e63bf09a5dc4ed857bd9fa46e14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"735a2a35f6b1c5e1b700cd4fe89f07201e4b403b4fd45e0a13a0acfcf9fe12f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"942360ff8169c2a19cebe856cddf17d577b93e9f7408dcd1cd1902c9daf1c0df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"ae9917932f90b2d8b738c6ed2ec73fd085b17fa53168e8719072bde1b10a9a0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"b0b84b753e0f678ce469ee4fc8400cad30e638a19fedbd6f7746aad6ef378b5a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"f2ec5a1a25d21e5dec2ed93eab92a26a91c080ad53e104d71fabc59db7c4d615","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3bfb93ffa2915664d7de20fc0b753c1a5307e53691a4b8098db3a7e206ca4906","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":29.166666666666668,"anchor_occurrences":7,"after_last_push_count":7,"also_in":["PROD"],"trend_counts":[0,0,1,1,2,2,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.002s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.002s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.000s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.000s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"55d4b1840d69212b2703b61209abbc219f27e5b689e1fef7a10a1d77779d4060","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"85944d4fa3cc8f9a02514637d6119a2fd69da50caa8d14fdae25e1e32d2d5f52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"55d4b1840d69212b2703b61209abbc219f27e5b689e1fef7a10a1d77779d4060","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"55d4b1840d69212b2703b61209abbc219f27e5b689e1fef7a10a1d77779d4060","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"20cd25cfb84069bf0dbeae90ccfea853127cb3d7c1e8d8f537a39512ce31cc19","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"20cd25cfb84069bf0dbeae90ccfea853127cb3d7c1e8d8f537a39512ce31cc19","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"55d4b1840d69212b2703b61209abbc219f27e5b689e1fef7a10a1d77779d4060","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":6,"runs_affected":5,"failed_runs":5,"run_impact_percent":20.833333333333332,"anchor_occurrences":6,"after_last_push_count":7,"also_in":["PROD"],"trend_counts":[0,3,2,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-20T08:35:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00118e480\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-20-h759ch\" in resourcegroup=\"rg-nightly-4-20-mzr4r5lz42qp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\",\n      \"name\": \"55ed743f-d106-4adb-b590-caa3369caeb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:25:48.884097042Z\",\n      \"endTime\": \"2026-08-24T23:49:50.709200875Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd7vm7aq4os1ksjghr3o1msj3ksno6l still exists (deletion dispatched at 2026-08-24T23:25:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00118e480\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-20-h759ch\" in resourcegroup=\"rg-nightly-4-20-mzr4r5lz42qp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\",\n      \"name\": \"55ed743f-d106-4adb-b590-caa3369caeb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:25:48.884097042Z\",\n      \"endTime\": \"2026-08-24T23:49:50.709200875Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd7vm7aq4os1ksjghr3o1msj3ksno6l still exists (deletion dispatched at 2026-08-24T23:25:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00111e108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-95b2mn6x5nc7\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56a5601c-84db-4c4c-afe3-fa090d79a425\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56a5601c-84db-4c4c-afe3-fa090d79a425\",\n      \"name\": \"56a5601c-84db-4c4c-afe3-fa090d79a425\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:48:30.429013476Z\",\n      \"endTime\": \"2026-08-20T10:12:32.646924727Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7kb8vek727d6pbio7effbv09ei0ua still exists (deletion dispatched at 2026-08-20T09:48:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00111e108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-95b2mn6x5nc7\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56a5601c-84db-4c4c-afe3-fa090d79a425\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56a5601c-84db-4c4c-afe3-fa090d79a425\",\n      \"name\": \"56a5601c-84db-4c4c-afe3-fa090d79a425\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:48:30.429013476Z\",\n      \"endTime\": \"2026-08-20T10:12:32.646924727Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7kb8vek727d6pbio7effbv09ei0ua still exists (deletion dispatched at 2026-08-20T09:48:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc002d34258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-x729kk\" in resourcegroup=\"rg-candidate-4-22-6wvs8lznc8fp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\",\n      \"name\": \"bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:47:07.957787635Z\",\n      \"endTime\": \"2026-08-20T10:11:12.923142399Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7h5aduc2uvihvfe2vdijl20h32glr still exists (deletion dispatched at 2026-08-20T09:47:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc002d34258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-x729kk\" in resourcegroup=\"rg-candidate-4-22-6wvs8lznc8fp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\",\n      \"name\": \"bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:47:07.957787635Z\",\n      \"endTime\": \"2026-08-20T10:11:12.923142399Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7h5aduc2uvihvfe2vdijl20h32glr still exists (deletion dispatched at 2026-08-20T09:47:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"6a0460b71f8ba99286143da10d9b371325b39609e9612fa043b5980a861407e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"3f5ac4cdd7ff19e9ff918b763fc044b6a568c83bba923b1245e9a1c9566d493d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"e0f6f930a179a3343bf2fd68ac2daa5ec05546590115b913738327ca0e47fe4d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"e605bb21ea5244e439cf62bcea1ffb88819d8256b4264429251651c7bd080039","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090159303129305088","occurred_at":"2026-08-19T19:29:45Z","signature_id":"0d48b115dcda077256c842a49dc3aa3b939af3e2623702d25a646ab55111bd97","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089990802548723712","occurred_at":"2026-08-19T08:20:12Z","signature_id":"1488dcf0a44d59096f0003c2bcb2ddab938abf19f850e64b113df3a7024a0e55","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":6,"runs_affected":4,"failed_runs":4,"run_impact_percent":16.666666666666668,"anchor_occurrences":6,"after_last_push_count":146,"also_in":["DEV","PROD"],"trend_counts":[0,0,4,1,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":13,"prior_last_seen_at":"2026-08-21T23:00:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e8ea40\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lf6jg45wtwxg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\",\n      \"name\": \"62db7451-86ec-4c6a-9227-bc775f891ebb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:05:01.131009318Z\",\n      \"endTime\": \"2026-08-24T23:24:05.637766062Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e8ea40\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lf6jg45wtwxg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\",\n      \"name\": \"62db7451-86ec-4c6a-9227-bc775f891ebb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:05:01.131009318Z\",\n      \"endTime\": \"2026-08-24T23:24:05.637766062Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3f2e0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-sh428kn5d89n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3af2b0d-5fad-4125-9764-31b214f8fee1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3af2b0d-5fad-4125-9764-31b214f8fee1\",\n      \"name\": \"b3af2b0d-5fad-4125-9764-31b214f8fee1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:04:23.850321515Z\",\n      \"endTime\": \"2026-08-21T23:23:28.45210372Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3f2e0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-sh428kn5d89n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3af2b0d-5fad-4125-9764-31b214f8fee1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3af2b0d-5fad-4125-9764-31b214f8fee1\",\n      \"name\": \"b3af2b0d-5fad-4125-9764-31b214f8fee1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:04:23.850321515Z\",\n      \"endTime\": \"2026-08-21T23:23:28.45210372Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-pt5scqn6n82h/cluster-candidate-4-21-28flhq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00124eb60\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-28flhq: failed waiting for cluster=\"cluster-candidate-4-21-28flhq\" in resourcegroup=\"rg-candidate-4-21-pt5scqn6n82h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/332f74a8-d0dc-4e4a-9737-70119dac90bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/332f74a8-d0dc-4e4a-9737-70119dac90bc\",\n      \"name\": \"332f74a8-d0dc-4e4a-9737-70119dac90bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T20:53:42.523371474Z\",\n      \"endTime\": \"2026-08-20T21:12:49.332578125Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-pt5scqn6n82h/cluster-candidate-4-21-28flhq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00124eb60\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-28flhq: failed waiting for cluster=\"cluster-candidate-4-21-28flhq\" in resourcegroup=\"rg-candidate-4-21-pt5scqn6n82h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/332f74a8-d0dc-4e4a-9737-70119dac90bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/332f74a8-d0dc-4e4a-9737-70119dac90bc\",\n      \"name\": \"332f74a8-d0dc-4e4a-9737-70119dac90bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T20:53:42.523371474Z\",\n      \"endTime\": \"2026-08-20T21:12:49.332578125Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"ce363ec1f1315846ac4ca18a663f8ca57451d8f5854815877a996fbdf3f45dd9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"fbf7d38b96f28f05b21778aab3674d4cce11c0c6f81909ef1d5dc73f06ad06e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090538506672672768","occurred_at":"2026-08-20T20:36:34Z","signature_id":"75e8134db7c38c2272b0135b9ef67b8e4ba756be9b98292cf44f0135c9355207","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"56f9bbbf6c1cff5e3a43f7b42723ff5d2896274197334b06141ae6c0de7f3f12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"7ebf3a7566f82ffac1f2fdc94973962a97b9cebbccee3a25837ecc58e0903fb4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"f2f000bdc64541602873ebc887a909036d077a88b2e4d46c3fb4c2246d3285bd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b6258cd488b080bf5ba7099ce8387b9d2f4d222d926f9a0c81022dc736e89db2","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":20.833333333333332,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,2,2,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017be510\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017be510\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001391020\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001391020\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013e8f30\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013e8f30\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"c14f9a5639d319819baa7a308cbb789519348e32b0abac6b02f5611b9187fbe7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"a4491e3bb9b86251a9e67c8b646d0760e0bef62043d0262fc0754c33958ef07f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"9aa9e14981ec5eaa3b7eaab007d9e3c7d2f7fd49eb9bb6e90dc9b41d9aebe244","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"bd404f1ba751bc9877a1e8fdc24c7640aff9f6d2b3d0bdcdce835db61939593c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"417bf2d786aadfe3f04e5550d280ceae37c25a8bf717f073a992b1ac95f1cad4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b936783c2d0b2a9d4ffabaeda14053cc9fef881b92be87e0e230002db2affbaf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":5,"runs_affected":4,"failed_runs":4,"run_impact_percent":16.666666666666668,"anchor_occurrences":5,"after_last_push_count":15,"also_in":["DEV"],"trend_counts":[1,5,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":12,"prior_last_seen_at":"2026-08-19T23:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c244b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-v62dhvmn5xjx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/433c7b9f-9ee6-405a-ac4b-61c2aba8b5b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/433c7b9f-9ee6-405a-ac4b-61c2aba8b5b8\",\n      \"name\": \"433c7b9f-9ee6-405a-ac4b-61c2aba8b5b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:23:38.105024916Z\",\n      \"endTime\": \"2026-08-19T23:47:40.969477415Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ugi7ah3g0hdhfnrdfe02s4641un92 still exists (deletion dispatched at 2026-08-19T23:23:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c244b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-v62dhvmn5xjx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/433c7b9f-9ee6-405a-ac4b-61c2aba8b5b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/433c7b9f-9ee6-405a-ac4b-61c2aba8b5b8\",\n      \"name\": \"433c7b9f-9ee6-405a-ac4b-61c2aba8b5b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:23:38.105024916Z\",\n      \"endTime\": \"2026-08-19T23:47:40.969477415Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ugi7ah3g0hdhfnrdfe02s4641un92 still exists (deletion dispatched at 2026-08-19T23:23:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013d62d0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-zk8j9gsrf47s\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83d822b3-ac2e-4cc4-966c-eab4ee8e8602\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83d822b3-ac2e-4cc4-966c-eab4ee8e8602\",\n      \"name\": \"83d822b3-ac2e-4cc4-966c-eab4ee8e8602\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T13:56:53.166820811Z\",\n      \"endTime\": \"2026-08-19T14:20:55.153951347Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9m491m1ahtrh178oqepopioqrgcdms still exists (deletion dispatched at 2026-08-19T13:56:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013d62d0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-zk8j9gsrf47s\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83d822b3-ac2e-4cc4-966c-eab4ee8e8602\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83d822b3-ac2e-4cc4-966c-eab4ee8e8602\",\n      \"name\": \"83d822b3-ac2e-4cc4-966c-eab4ee8e8602\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T13:56:53.166820811Z\",\n      \"endTime\": \"2026-08-19T14:20:55.153951347Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9m491m1ahtrh178oqepopioqrgcdms still exists (deletion dispatched at 2026-08-19T13:56:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00112a408\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-rns7wq\" in resourcegroup=\"rg-zstream-upgrade-4-20-vhxh5b6zszlg\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1928d5f-397e-4b68-89ec-79ef6c7cd4ac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1928d5f-397e-4b68-89ec-79ef6c7cd4ac\",\n      \"name\": \"e1928d5f-397e-4b68-89ec-79ef6c7cd4ac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T14:13:37.892189859Z\",\n      \"endTime\": \"2026-08-19T14:37:44.990617474Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9mivqf2eovoc0ctilj3pgriaobnqh1 still exists (deletion dispatched at 2026-08-19T14:13:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00112a408\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-rns7wq\" in resourcegroup=\"rg-zstream-upgrade-4-20-vhxh5b6zszlg\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1928d5f-397e-4b68-89ec-79ef6c7cd4ac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1928d5f-397e-4b68-89ec-79ef6c7cd4ac\",\n      \"name\": \"e1928d5f-397e-4b68-89ec-79ef6c7cd4ac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T14:13:37.892189859Z\",\n      \"endTime\": \"2026-08-19T14:37:44.990617474Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9mivqf2eovoc0ctilj3pgriaobnqh1 still exists (deletion dispatched at 2026-08-19T14:13:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"31c34b662aa4007e0dd4d1ce9746adfafd06942edf2f371c9d1adf52e6ff6914","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090064577516539904","occurred_at":"2026-08-19T13:13:21Z","signature_id":"0b849c1b1930080096c80502fde35ce7acebfcf86d1a355be93bfad04a63ff81","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090064577516539904","occurred_at":"2026-08-19T13:13:21Z","signature_id":"9042ad9b49db684e855010022007c6f95a287cf403ee1d9dbdfb1c6014061aa8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089951420068401152","occurred_at":"2026-08-19T05:43:42Z","signature_id":"e5a5a72236358d122a4c716a2585811976fccb0e8b9b1d216e22cf2352d6eb1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089895148551933952","occurred_at":"2026-08-19T02:00:06Z","signature_id":"d0b9bf7336ba6c5e0c7514c62a1701c85355bf80d65b2d77b4cf498038933f28","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"c75b40567502299dd397c0248cce6097e2377df763d3deec7ae8c7f9cd09e3b2","failure_pattern":"node pool \u003cnodepool\u003e verification failed: \u003cnode\u003e (no version in containerRuntimeVersion \"cri-o://1","search_query":"node pool upgrade verification failed: g2a9g8l9y5u0b6x-nps-420-422-96vnb-jdgtr (no version in containerRuntimeVersion \"cri-o://1","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":16.666666666666668,"anchor_occurrences":4,"after_last_push_count":4,"also_in":["PROD"],"trend_counts":[0,0,0,1,1,1,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0012c0d10\u003e: \n    node pool upgrade verification failed: w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]},{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0012c0d10\u003e: \n    node pool upgrade verification failed: w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]},{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002a14f40\u003e: \n    node pool upgrade verification failed: x2t8p7p8m2s5y4s-nps-420-422-mqm7r-2pww5 (no version in containerRuntimeVersion \"cri-o://1.35.4\"); x2t8p7p8m2s5y4s-nps-420-422-mqm7r-vvlrc (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"x2t8p7p8m2s5y4s-nps-420-422-mqm7r-2pww5\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"x2t8p7p8m2s5y4s-nps-420-422-mqm7r-vvlrc\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002a14f40\u003e: \n    node pool upgrade verification failed: x2t8p7p8m2s5y4s-nps-420-422-mqm7r-2pww5 (no version in containerRuntimeVersion \"cri-o://1.35.4\"); x2t8p7p8m2s5y4s-nps-420-422-mqm7r-vvlrc (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"x2t8p7p8m2s5y4s-nps-420-422-mqm7r-2pww5\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"x2t8p7p8m2s5y4s-nps-420-422-mqm7r-vvlrc\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002bd0580\u003e: \n    node pool upgrade verification failed: g2a9g8l9y5u0b6x-nps-420-422-96vnb-jdgtr (no version in containerRuntimeVersion \"cri-o://1.35.4\"); g2a9g8l9y5u0b6x-nps-420-422-96vnb-tl58m (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"g2a9g8l9y5u0b6x-nps-420-422-96vnb-jdgtr\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"g2a9g8l9y5u0b6x-nps-420-422-96vnb-tl58m\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002bd0580\u003e: \n    node pool upgrade verification failed: g2a9g8l9y5u0b6x-nps-420-422-96vnb-jdgtr (no version in containerRuntimeVersion \"cri-o://1.35.4\"); g2a9g8l9y5u0b6x-nps-420-422-96vnb-tl58m (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"g2a9g8l9y5u0b6x-nps-420-422-96vnb-jdgtr\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"g2a9g8l9y5u0b6x-nps-420-422-96vnb-tl58m\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"5565031e089c6eb68fe43eb98d2d43776a82305498e8c6a0a7f4c918b0348bf0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"6cb20baa379b7fcdcba1d14b4a55e6c86cfdee72a1c93591debfc4dfe3e0724b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"749b338f0f666d9846dc55dd5356b693a9e5f5c64374dd55730a6f912a7a0323","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"9f5e26191b5a76b3738feae66a80838e9ebca5f7e61655121cd70c58e39a58ff","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"dc664af01d9bca7cbb1a073b6ebc6b372492bdba45274f79dc51fffb40636bd6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":12.5,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["DEV","PROD"],"trend_counts":[0,1,2,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-20T08:35:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de13e0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-2kbvdf4gpb7z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ea7be1d-979f-43d0-a6eb-4fadf5b52158\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ea7be1d-979f-43d0-a6eb-4fadf5b52158\",\n      \"name\": \"1ea7be1d-979f-43d0-a6eb-4fadf5b52158\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:49:43.253613436Z\",\n      \"endTime\": \"2026-08-20T10:08:52.449510653Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de13e0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-2kbvdf4gpb7z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ea7be1d-979f-43d0-a6eb-4fadf5b52158\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ea7be1d-979f-43d0-a6eb-4fadf5b52158\",\n      \"name\": \"1ea7be1d-979f-43d0-a6eb-4fadf5b52158\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:49:43.253613436Z\",\n      \"endTime\": \"2026-08-20T10:08:52.449510653Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000678260\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-lvmrprf9szxn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc452150-a44c-4cdf-95f9-3ab1762bc659\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc452150-a44c-4cdf-95f9-3ab1762bc659\",\n      \"name\": \"dc452150-a44c-4cdf-95f9-3ab1762bc659\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T03:43:58.925008531Z\",\n      \"endTime\": \"2026-08-20T04:03:00.154367927Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000678260\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-lvmrprf9szxn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc452150-a44c-4cdf-95f9-3ab1762bc659\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc452150-a44c-4cdf-95f9-3ab1762bc659\",\n      \"name\": \"dc452150-a44c-4cdf-95f9-3ab1762bc659\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T03:43:58.925008531Z\",\n      \"endTime\": \"2026-08-20T04:03:00.154367927Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ff4e60\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-l88qjllvzfng\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c412073-0e2e-4d8b-bffc-e50b877db2c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c412073-0e2e-4d8b-bffc-e50b877db2c8\",\n      \"name\": \"3c412073-0e2e-4d8b-bffc-e50b877db2c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T14:26:29.558969897Z\",\n      \"endTime\": \"2026-08-19T14:45:35.6247326Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ff4e60\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-l88qjllvzfng\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c412073-0e2e-4d8b-bffc-e50b877db2c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c412073-0e2e-4d8b-bffc-e50b877db2c8\",\n      \"name\": \"3c412073-0e2e-4d8b-bffc-e50b877db2c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T14:26:29.558969897Z\",\n      \"endTime\": \"2026-08-19T14:45:35.6247326Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"1fe08479e59e5ed4afa0e1f161e992bf328632b651a998809d33c2ece866d07c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090276754840096768","occurred_at":"2026-08-20T03:16:28Z","signature_id":"7e2cf5d1bff2be054d231178fc2524db6c79b0da8fc042c6972e38c1b9001d27","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090064577516539904","occurred_at":"2026-08-19T13:13:21Z","signature_id":"a8c846c9bd4c317ad39a1011330ed9d1aa7ef3ae4b2a3224a56dbb4bb7373821","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7e3c211098fa13aa2f965e23e8e58e902c8d4f3b340bbbf0b11eb0cf36bda7ed","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":12.5,"anchor_occurrences":3,"after_last_push_count":11,"also_in":["DEV","PROD"],"trend_counts":[1,0,2,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":10,"prior_last_seen_at":"2026-08-21T23:00:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:139]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006ae100\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-s82wq8qv974g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/389f219e-5f8d-4b6a-9c72-fdd66be0b32d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/389f219e-5f8d-4b6a-9c72-fdd66be0b32d\",\n      \"name\": \"389f219e-5f8d-4b6a-9c72-fdd66be0b32d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:17:03.253121618Z\",\n      \"endTime\": \"2026-08-21T23:36:09.200090899Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:139]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006ae100\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-s82wq8qv974g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/389f219e-5f8d-4b6a-9c72-fdd66be0b32d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/389f219e-5f8d-4b6a-9c72-fdd66be0b32d\",\n      \"name\": \"389f219e-5f8d-4b6a-9c72-fdd66be0b32d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:17:03.253121618Z\",\n      \"endTime\": \"2026-08-21T23:36:09.200090899Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-21 with version 4.21.28\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010bc7e0\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-8mw8p7\" in resourcegroup=\"rg-np-version-upgrade-8mw8p7-jzxbk7cd5ktl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/217c5b5b-876e-4dfd-aa27-a2f787a06b7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/217c5b5b-876e-4dfd-aa27-a2f787a06b7d\",\n      \"name\": \"217c5b5b-876e-4dfd-aa27-a2f787a06b7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:52:42.575458861Z\",\n      \"endTime\": \"2026-08-20T10:11:42.477430185Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-21 with version 4.21.28\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010bc7e0\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-8mw8p7\" in resourcegroup=\"rg-np-version-upgrade-8mw8p7-jzxbk7cd5ktl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/217c5b5b-876e-4dfd-aa27-a2f787a06b7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/217c5b5b-876e-4dfd-aa27-a2f787a06b7d\",\n      \"name\": \"217c5b5b-876e-4dfd-aa27-a2f787a06b7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:52:42.575458861Z\",\n      \"endTime\": \"2026-08-20T10:11:42.477430185Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009eaaa0\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qjg88gwvq5xj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2facd99-ace4-452a-af16-9d4629a551d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2facd99-ace4-452a-af16-9d4629a551d4\",\n      \"name\": \"c2facd99-ace4-452a-af16-9d4629a551d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T04:35:11.139884226Z\",\n      \"endTime\": \"2026-08-20T04:54:12.658497794Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009eaaa0\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qjg88gwvq5xj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2facd99-ace4-452a-af16-9d4629a551d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2facd99-ace4-452a-af16-9d4629a551d4\",\n      \"name\": \"c2facd99-ace4-452a-af16-9d4629a551d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T04:35:11.139884226Z\",\n      \"endTime\": \"2026-08-20T04:54:12.658497794Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"463a01fef05dd95fc4804e0b40a98b4138b6a654495626a1d5814204f1feb3ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"30e3ce44a70a32080f494a46962fb9e3504a602768cbec5e3cc893e94e06d32f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090276754840096768","occurred_at":"2026-08-20T03:16:28Z","signature_id":"e14f3f9d9bbacbaff6e558f257e54562c56eab915ed893444a9648f95512b4c2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d58c4430f859260715a2df79338de096b1f7324f15367436040e29b8a0d491e6","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":8.333333333333334,"anchor_occurrences":3,"after_last_push_count":6,"also_in":["DEV","INT","PROD"],"trend_counts":[0,0,0,1,2,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-22T02:00:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000886800\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-7qtvldntpktk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ad306216-5622-4121-8bf1-f015261fa044\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ad306216-5622-4121-8bf1-f015261fa044\",\n      \"name\": \"ad306216-5622-4121-8bf1-f015261fa044\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T02:03:58.055302295Z\",\n      \"endTime\": \"2026-08-22T02:23:05.756072788Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000886800\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-7qtvldntpktk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ad306216-5622-4121-8bf1-f015261fa044\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ad306216-5622-4121-8bf1-f015261fa044\",\n      \"name\": \"ad306216-5622-4121-8bf1-f015261fa044\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T02:03:58.055302295Z\",\n      \"endTime\": \"2026-08-22T02:23:05.756072788Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000517020\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9zbjjzhbwsc7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\",\n      \"name\": \"8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T02:03:58.196286523Z\",\n      \"endTime\": \"2026-08-22T02:23:05.606457627Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000517020\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9zbjjzhbwsc7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\",\n      \"name\": \"8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T02:03:58.196286523Z\",\n      \"endTime\": \"2026-08-22T02:23:05.606457627Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008e61c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gdl8mt5sbkqm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4841240d-e2f8-4813-8ee4-75401f65cda9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4841240d-e2f8-4813-8ee4-75401f65cda9\",\n      \"name\": \"4841240d-e2f8-4813-8ee4-75401f65cda9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:05:01.793828852Z\",\n      \"endTime\": \"2026-08-21T02:24:02.290641543Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008e61c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gdl8mt5sbkqm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4841240d-e2f8-4813-8ee4-75401f65cda9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4841240d-e2f8-4813-8ee4-75401f65cda9\",\n      \"name\": \"4841240d-e2f8-4813-8ee4-75401f65cda9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:05:01.793828852Z\",\n      \"endTime\": \"2026-08-21T02:24:02.290641543Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"aa7e5f753e6e6e31752d40a0be892d0d8394c3a4ed7f539a58571438102dcd83","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"da606e988e3b163984f1144b1c2c0f76f91f7a639869fb04b4b28da971b69c0d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"90540ad1f71b81bed938e2d289c44e688fd651904bd2cb8cddef646e47d0d371","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"de0797a28411af3d24415a55d3d3897578493557d5bbf06af33b5687d5423fc9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":8.333333333333334,"anchor_occurrences":2,"after_last_push_count":3,"also_in":["DEV","PROD"],"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-20T12:36:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-4r45c4hp7cql/cluster-candidate-4-20-r96znq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008fa700\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-r96znq: failed waiting for cluster=\"cluster-candidate-4-20-r96znq\" in resourcegroup=\"rg-candidate-4-20-4r45c4hp7cql\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3742f6f7-818c-4bbe-a0c4-ca70030eb73e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3742f6f7-818c-4bbe-a0c4-ca70030eb73e\",\n      \"name\": \"3742f6f7-818c-4bbe-a0c4-ca70030eb73e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:15:18.696546382Z\",\n      \"endTime\": \"2026-08-20T13:34:20.875186386Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-4r45c4hp7cql/cluster-candidate-4-20-r96znq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008fa700\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-r96znq: failed waiting for cluster=\"cluster-candidate-4-20-r96znq\" in resourcegroup=\"rg-candidate-4-20-4r45c4hp7cql\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3742f6f7-818c-4bbe-a0c4-ca70030eb73e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3742f6f7-818c-4bbe-a0c4-ca70030eb73e\",\n      \"name\": \"3742f6f7-818c-4bbe-a0c4-ca70030eb73e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:15:18.696546382Z\",\n      \"endTime\": \"2026-08-20T13:34:20.875186386Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-gpmtwfgcr6xn/cluster-candidate-4-20-6mrg8j should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001090160\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-6mrg8j: failed waiting for cluster=\"cluster-candidate-4-20-6mrg8j\" in resourcegroup=\"rg-candidate-4-20-gpmtwfgcr6xn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1f59ecc-759d-4b6c-8b93-932b0208adf4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1f59ecc-759d-4b6c-8b93-932b0208adf4\",\n      \"name\": \"a1f59ecc-759d-4b6c-8b93-932b0208adf4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T08:53:33.518746975Z\",\n      \"endTime\": \"2026-08-20T09:12:39.900608486Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-gpmtwfgcr6xn/cluster-candidate-4-20-6mrg8j should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001090160\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-6mrg8j: failed waiting for cluster=\"cluster-candidate-4-20-6mrg8j\" in resourcegroup=\"rg-candidate-4-20-gpmtwfgcr6xn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1f59ecc-759d-4b6c-8b93-932b0208adf4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1f59ecc-759d-4b6c-8b93-932b0208adf4\",\n      \"name\": \"a1f59ecc-759d-4b6c-8b93-932b0208adf4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T08:53:33.518746975Z\",\n      \"endTime\": \"2026-08-20T09:12:39.900608486Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"382e53c6f5440687f3efd2f0f735d848f4f2a75c27dbddc1aa370d48842e5ac7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"3aeb591de359ede5d82e500a19257072d85b9193e57ac8f4a13b125e2c1b0f69","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"a146c4d50a72c43e4a69c21e18547ba5a2a0a2b53d6985079637bcec0f5acdda","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":8.333333333333334,"anchor_occurrences":2,"after_last_push_count":11,"also_in":["DEV","INT","PROD"],"trend_counts":[3,2,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-19T23:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c54180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8hvpj6vtkqlf\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d19e078c-f7b8-4be7-ba0d-0921f4e3a585\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d19e078c-f7b8-4be7-ba0d-0921f4e3a585\",\n      \"name\": \"d19e078c-f7b8-4be7-ba0d-0921f4e3a585\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:47:16.516011942Z\",\n      \"endTime\": \"2026-08-20T00:11:22.445943521Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9uhn33u6bvucvlivpurd7qbjud4jvg still exists (deletion dispatched at 2026-08-19T23:47:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c54180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8hvpj6vtkqlf\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d19e078c-f7b8-4be7-ba0d-0921f4e3a585\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d19e078c-f7b8-4be7-ba0d-0921f4e3a585\",\n      \"name\": \"d19e078c-f7b8-4be7-ba0d-0921f4e3a585\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:47:16.516011942Z\",\n      \"endTime\": \"2026-08-20T00:11:22.445943521Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9uhn33u6bvucvlivpurd7qbjud4jvg still exists (deletion dispatched at 2026-08-19T23:47:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017d6138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-ingress\" in resourcegroup=\"private-ingress-qxfhsjtcs5wr\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c2f0336-078f-4209-9e4d-a24e8e832089\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c2f0336-078f-4209-9e4d-a24e8e832089\",\n      \"name\": \"2c2f0336-078f-4209-9e4d-a24e8e832089\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:31:15.704278812Z\",\n      \"endTime\": \"2026-08-19T02:55:20.050088368Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9c23lo4u800am915f4oldars1ighvd still exists (deletion dispatched at 2026-08-19T02:31:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017d6138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-ingress\" in resourcegroup=\"private-ingress-qxfhsjtcs5wr\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c2f0336-078f-4209-9e4d-a24e8e832089\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c2f0336-078f-4209-9e4d-a24e8e832089\",\n      \"name\": \"2c2f0336-078f-4209-9e4d-a24e8e832089\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:31:15.704278812Z\",\n      \"endTime\": \"2026-08-19T02:55:20.050088368Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9c23lo4u800am915f4oldars1ighvd still exists (deletion dispatched at 2026-08-19T02:31:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"b0ce1bdb91a35682a29122a0fcb2cafab1e41540901b85ad9cd097c5e00d92f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089895148551933952","occurred_at":"2026-08-19T02:00:06Z","signature_id":"c272e40d8f4e2729377a63b8f6a0cf17d1726c519963fbcdae85d0d65393f934","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6fc45abab45bac910821b9bf649e0a8d2f5cae8a2886fe2e7857c4e0907eb987","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["DEV"],"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-5jbtkc\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00147aa60\u003e: \n    failed to create HCP cluster np-noedge-5jbtkc: failed waiting for cluster=\"np-noedge-5jbtkc\" in resourcegroup=\"rg-np-noedge-5jbtkc-5kwghxcpjkf6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/edc3030e-e575-42e1-8662-b09b51002c15\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/edc3030e-e575-42e1-8662-b09b51002c15\",\n      \"name\": \"edc3030e-e575-42e1-8662-b09b51002c15\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:45:54.462061981Z\",\n      \"endTime\": \"2026-08-21T03:04:54.752602351Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, ingress-operator, cluster-image-registry-operator, catalog-operator, dns-operator, packageserver, redhat-operators-catalog, community-operators-catalog, cluster-version-operator, cluster-policy-controller, certified-operators-catalog, openshift-route-controller-manager, csi-snapshot-controller-operator, redhat-marketplace-catalog, cluster-network-operator, olm-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-5jbtkc\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00147aa60\u003e: \n    failed to create HCP cluster np-noedge-5jbtkc: failed waiting for cluster=\"np-noedge-5jbtkc\" in resourcegroup=\"rg-np-noedge-5jbtkc-5kwghxcpjkf6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/edc3030e-e575-42e1-8662-b09b51002c15\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/edc3030e-e575-42e1-8662-b09b51002c15\",\n      \"name\": \"edc3030e-e575-42e1-8662-b09b51002c15\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:45:54.462061981Z\",\n      \"endTime\": \"2026-08-21T03:04:54.752602351Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, ingress-operator, cluster-image-registry-operator, catalog-operator, dns-operator, packageserver, redhat-operators-catalog, community-operators-catalog, cluster-version-operator, cluster-policy-controller, certified-operators-catalog, openshift-route-controller-manager, csi-snapshot-controller-operator, redhat-marketplace-catalog, cluster-network-operator, olm-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005fe2a0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-ttg7gz2lhlcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ba59ea70-9b35-4783-a0ec-868e44666cde\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ba59ea70-9b35-4783-a0ec-868e44666cde\",\n      \"name\": \"ba59ea70-9b35-4783-a0ec-868e44666cde\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:50:23.471170015Z\",\n      \"endTime\": \"2026-08-21T03:09:24.769917396Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, redhat-operators-catalog, dns-operator, catalog-operator, cluster-image-registry-operator, community-operators-catalog, openshift-route-controller-manager, olm-operator, cluster-policy-controller, certified-operators-catalog, cluster-version-operator, openshift-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005fe2a0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-ttg7gz2lhlcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ba59ea70-9b35-4783-a0ec-868e44666cde\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ba59ea70-9b35-4783-a0ec-868e44666cde\",\n      \"name\": \"ba59ea70-9b35-4783-a0ec-868e44666cde\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:50:23.471170015Z\",\n      \"endTime\": \"2026-08-21T03:09:24.769917396Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, redhat-operators-catalog, dns-operator, catalog-operator, cluster-image-registry-operator, community-operators-catalog, openshift-route-controller-manager, olm-operator, cluster-policy-controller, certified-operators-catalog, cluster-version-operator, openshift-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"2b76ccfe8aac8f29923c772019c68b4d1daa538e7e3fedec8fb0b2f71b8b486f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"91e88f33d22326cd1e1f28e5cf546d8a5f825f85b45e6974877ac8761f301bbc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3b51c13a3cafe6196693705cdaea38d26216b0eac963a598136ad9cacb24071b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["DEV"],"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-m6dffq9dg6px/cluster-candidate-4-22-rttbvz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fee820\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-rttbvz: failed waiting for cluster=\"cluster-candidate-4-22-rttbvz\" in resourcegroup=\"rg-candidate-4-22-m6dffq9dg6px\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6621367e-101e-4d5a-ab22-9e61d95865dd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6621367e-101e-4d5a-ab22-9e61d95865dd\",\n      \"name\": \"6621367e-101e-4d5a-ab22-9e61d95865dd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:29:22.397457374Z\",\n      \"endTime\": \"2026-08-21T02:48:23.657141881Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-m6dffq9dg6px/cluster-candidate-4-22-rttbvz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fee820\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-rttbvz: failed waiting for cluster=\"cluster-candidate-4-22-rttbvz\" in resourcegroup=\"rg-candidate-4-22-m6dffq9dg6px\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6621367e-101e-4d5a-ab22-9e61d95865dd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6621367e-101e-4d5a-ab22-9e61d95865dd\",\n      \"name\": \"6621367e-101e-4d5a-ab22-9e61d95865dd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:29:22.397457374Z\",\n      \"endTime\": \"2026-08-21T02:48:23.657141881Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-dqp9mm5cksqn/cluster-candidate-4-20-bfprqt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015cf400\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-bfprqt: failed waiting for cluster=\"cluster-candidate-4-20-bfprqt\" in resourcegroup=\"rg-candidate-4-20-dqp9mm5cksqn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e423bc99-f4f6-4437-a051-cbb783aa9c19\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e423bc99-f4f6-4437-a051-cbb783aa9c19\",\n      \"name\": \"e423bc99-f4f6-4437-a051-cbb783aa9c19\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:18.115131874Z\",\n      \"endTime\": \"2026-08-21T02:22:22.541705225Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-dqp9mm5cksqn/cluster-candidate-4-20-bfprqt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015cf400\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-bfprqt: failed waiting for cluster=\"cluster-candidate-4-20-bfprqt\" in resourcegroup=\"rg-candidate-4-20-dqp9mm5cksqn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e423bc99-f4f6-4437-a051-cbb783aa9c19\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e423bc99-f4f6-4437-a051-cbb783aa9c19\",\n      \"name\": \"e423bc99-f4f6-4437-a051-cbb783aa9c19\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:18.115131874Z\",\n      \"endTime\": \"2026-08-21T02:22:22.541705225Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"854ad62699d22398160556ff1d1461a048fa03f78571ec0dcf40580c9a2fd02d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"b3374a6a172ebb403ac686e50368e308458096eea2258fdb839e7b2281ad2231","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"bae2c31dfd388275f409339683e6035aa6871992170a42fd34eae18131b6e2ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["DEV","INT","PROD"],"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-h762z2 with version 4.21.29\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000916f80\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-h762z2: failed waiting for cluster=\"np-version-upgrade-cluster-h762z2\" in resourcegroup=\"rg-np-version-upgrade-h762z2-227jm54s5fnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a8fe4db-8a24-46bf-99be-8771ce79a814\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a8fe4db-8a24-46bf-99be-8771ce79a814\",\n      \"name\": \"7a8fe4db-8a24-46bf-99be-8771ce79a814\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:45:47.941159859Z\",\n      \"endTime\": \"2026-08-21T03:04:54.892784877Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-h762z2 with version 4.21.29\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000916f80\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-h762z2: failed waiting for cluster=\"np-version-upgrade-cluster-h762z2\" in resourcegroup=\"rg-np-version-upgrade-h762z2-227jm54s5fnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a8fe4db-8a24-46bf-99be-8771ce79a814\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a8fe4db-8a24-46bf-99be-8771ce79a814\",\n      \"name\": \"7a8fe4db-8a24-46bf-99be-8771ce79a814\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:45:47.941159859Z\",\n      \"endTime\": \"2026-08-21T03:04:54.892784877Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c2e7a0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dhdl29xp87ff\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\",\n      \"name\": \"94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:39:56.502029488Z\",\n      \"endTime\": \"2026-08-21T02:59:03.450785892Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c2e7a0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dhdl29xp87ff\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\",\n      \"name\": \"94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:39:56.502029488Z\",\n      \"endTime\": \"2026-08-21T02:59:03.450785892Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"021ad8856d5c396fba67aba9c2c2ff60d51660b492523606a295b59e5f5dfb89","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"9b7bbad63103e55151b58e9fe0aa82ebcd0a55b9322b35d817145a7e77a3fb08","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"ab5d80437500581ad7ce3942572aca8a7c5ea9eda26f4367215f73be1501d7a0","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-l7xnjldf8s7s\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-l7xnjldf8s7s, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["DEV"],"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:228]: failed to update node pool np-1 with new label and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002a1d1a0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-l7xnjldf8s7s\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-l7xnjldf8s7s, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:228]: failed to update node pool np-1 with new label and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002a1d1a0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-l7xnjldf8s7s\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-l7xnjldf8s7s, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:196]: failed to upgrade node pool npupgrade-4-20 to version 4.21.0-0.nightly-multi-2026-08-20-065923\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002d08a80\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-z5466s\" resourcegroup=\"rg-np-version-upgrade-z5466s-wkz757zpgbsr\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-z5466s in resource group rg-np-version-upgrade-z5466s-wkz757zpgbsr, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:196]: failed to upgrade node pool npupgrade-4-20 to version 4.21.0-0.nightly-multi-2026-08-20-065923\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002d08a80\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-z5466s\" resourcegroup=\"rg-np-version-upgrade-z5466s-wkz757zpgbsr\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-z5466s in resource group rg-np-version-upgrade-z5466s-wkz757zpgbsr, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"bef58880af3f67335fb38c9962fdc80b164c51531808fd3a944b58b5abc7835e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"fdab187cfe558b4bb3f41aa7470064acf21835a87bb5df9e4d6a394f761d1b45","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"f170a7927803930d63ef797276984cf62fcbd2f07de426245306d08da19e2bb9","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000b06150\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-20-052229, targetMinor=4.21.0-0.nightly-multi-2026-08-20-065923) failed: failed to get clusterversion \"version\": Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000b06150\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-20-052229, targetMinor=4.21.0-0.nightly-multi-2026-08-20-065923) failed: failed to get clusterversion \"version\": Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f0c138\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-20-141954, targetMinor=4.23.0-0.nightly-multi-2026-08-20-085020) failed: failed to get clusterversion \"version\": Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f0c138\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-20-141954, targetMinor=4.23.0-0.nightly-multi-2026-08-20-085020) failed: failed to get clusterversion \"version\": Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"ee2ed8274c95a8959d110687ae9ece838bb447f44f7c22e5c06d8ad3a3a7f27b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"ee48f986fd9e592059d2a5cd09e88c38d974e8948f1e0f0065c5a8641bfef3cd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d8e13d55daacf4ddf74bdc040314e449b308767e96d40780e45800466f812928","failure_pattern":"list nodes (nodePool=\u003cnodepool\u003e): Get \"\u003curl\u003e","search_query":"list nodes (nodePool=npdg-422-420): Get \"https://api","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:772]: Timed out after 2700.000s.\nnode pool nodes were not recreated at downgrade target version 4.20.0-0.nightly-multi-2026-08-20-052229\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000df20a0\u003e: \n    list nodes (nodePool=npdg-422-420): Get \"https://api.d1i3c9v1k0g4b5v.7gi9.uksouthstaging.aroapp-hcp.io:443/api/v1/nodes\": net/http: TLS handshake timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:772]: Timed out after 2700.000s.\nnode pool nodes were not recreated at downgrade target version 4.20.0-0.nightly-multi-2026-08-20-052229\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000df20a0\u003e: \n    list nodes (nodePool=npdg-422-420): Get \"https://api.d1i3c9v1k0g4b5v.7gi9.uksouthstaging.aroapp-hcp.io:443/api/v1/nodes\": net/http: TLS handshake timeout\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0010e2920\u003e: \n    list nodes (nodePool=nps-420-422): Get \"https://api.m1w5a4x7b2n1l4i.t4j1.uksouthstaging.aroapp-hcp.io:443/api/v1/nodes\": EOF\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0010e2920\u003e: \n    list nodes (nodePool=nps-420-422): Get \"https://api.m1w5a4x7b2n1l4i.t4j1.uksouthstaging.aroapp-hcp.io:443/api/v1/nodes\": EOF\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"3519eaf3902036b2df340ad7c69439459fca96026f79aacac0e4ada2e8cb9ab4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"846c6382df9acef0fa5ddc2a6d1f4dc3f653f6d1f1033da3dc950698cda0412f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d6ec87c7d972149dae5ab13de0394af8c0dc25683fd32e5ecca933f42f5cfc45","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-z2b7swxgrmhl, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:146]: failed to delete customer resource group \"cx-rg-cluster-z2b7swxgrmhl\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0039aa420\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-z2b7swxgrmhl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:146]: failed to delete customer resource group \"cx-rg-cluster-z2b7swxgrmhl\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0039aa420\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-z2b7swxgrmhl, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"96a2fe2e929f73bfb9adc313d26af9eac1da35f5933f0823653985b4fdc7f529","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"90cdb36cc9a54567f53aaac3f8c671c2433ddd6b58320efddb4af3f213c50608","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011a6420\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wpsgmw5rph44\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb23f88c-ec78-4c60-8fbf-d5847d914873\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb23f88c-ec78-4c60-8fbf-d5847d914873\",\n      \"name\": \"eb23f88c-ec78-4c60-8fbf-d5847d914873\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:05:11.172903386Z\",\n      \"endTime\": \"2026-08-21T02:24:12.542645954Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011a6420\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wpsgmw5rph44\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb23f88c-ec78-4c60-8fbf-d5847d914873\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb23f88c-ec78-4c60-8fbf-d5847d914873\",\n      \"name\": \"eb23f88c-ec78-4c60-8fbf-d5847d914873\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:05:11.172903386Z\",\n      \"endTime\": \"2026-08-21T02:24:12.542645954Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"7fb8dc9dd6caa75c82f132d73240ca1b73d2c2ed70d78cc29ea092d7fc3ce43b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6cef856251578d6d4f842b0ea51927280752396355aaa78eb5a3ffb310ef3d92","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:140]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-52rvxg\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018c2cc0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-52rvxg: failed waiting for cluster=\"cp-ystream-upgrade-4-22-52rvxg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-sht9lcl99cmr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\",\n      \"name\": \"ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:33:22.674679438Z\",\n      \"endTime\": \"2026-08-21T02:52:23.424354393Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-storage-operator, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, kube-controller-manager, kube-scheduler, cluster-policy-controller, csi-snapshot-controller-operator, cluster-version-operator, openshift-route-controller-manager, catalog-operator, openshift-controller-manager, cluster-node-tuning-operator, dns-operator, certified-operators-catalog, hosted-cluster-config-operator, cluster-network-operator, olm-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:140]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-52rvxg\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018c2cc0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-52rvxg: failed waiting for cluster=\"cp-ystream-upgrade-4-22-52rvxg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-sht9lcl99cmr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\",\n      \"name\": \"ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:33:22.674679438Z\",\n      \"endTime\": \"2026-08-21T02:52:23.424354393Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-storage-operator, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, kube-controller-manager, kube-scheduler, cluster-policy-controller, csi-snapshot-controller-operator, cluster-version-operator, openshift-route-controller-manager, catalog-operator, openshift-controller-manager, cluster-node-tuning-operator, dns-operator, certified-operators-catalog, hosted-cluster-config-operator, cluster-network-operator, olm-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"a653dd0fe3070ac575df986a0df9a1327f00fa60786168762feb747e19bdb7f7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7356cf97f840631ab4a0a63ce4f36ca842474a0ca4160d56ce50ad927de67954","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T08:35:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-sttt5r at version 4.22.10\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f130e0\u003e: \n    failed to create HCP cluster np-skip-minor-sttt5r: failed waiting for cluster=\"np-skip-minor-sttt5r\" in resourcegroup=\"rg-np-skip-minor-sttt5r-4fsvkfwk5vdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a570b418-a849-489b-8b7e-8e8fb8135baa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a570b418-a849-489b-8b7e-8e8fb8135baa\",\n      \"name\": \"a570b418-a849-489b-8b7e-8e8fb8135baa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:36:05.972495231Z\",\n      \"endTime\": \"2026-08-20T09:55:11.507838131Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, azure-cloud-controller-manager, dns-operator, olm-operator, machine-approver, kube-controller-manager, hosted-cluster-config-operator, kube-storage-version-migrator, catalog-operator, olm-collect-profiles, kube-scheduler, community-operators-catalog, cluster-autoscaler, openshift-route-controller-manager, cluster-policy-controller, csi-snapshot-controller-operator, packageserver, control-plane-pki-operator, cluster-storage-operator, ingress-operator, konnectivity-agent, redhat-operators-catalog, openshift-apiserver, cluster-image-registry-operator, redhat-marketplace-catalog, certified-operators-catalog, openshift-controller-manager, cluster-network-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-sttt5r at version 4.22.10\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f130e0\u003e: \n    failed to create HCP cluster np-skip-minor-sttt5r: failed waiting for cluster=\"np-skip-minor-sttt5r\" in resourcegroup=\"rg-np-skip-minor-sttt5r-4fsvkfwk5vdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a570b418-a849-489b-8b7e-8e8fb8135baa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a570b418-a849-489b-8b7e-8e8fb8135baa\",\n      \"name\": \"a570b418-a849-489b-8b7e-8e8fb8135baa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:36:05.972495231Z\",\n      \"endTime\": \"2026-08-20T09:55:11.507838131Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, azure-cloud-controller-manager, dns-operator, olm-operator, machine-approver, kube-controller-manager, hosted-cluster-config-operator, kube-storage-version-migrator, catalog-operator, olm-collect-profiles, kube-scheduler, community-operators-catalog, cluster-autoscaler, openshift-route-controller-manager, cluster-policy-controller, csi-snapshot-controller-operator, packageserver, control-plane-pki-operator, cluster-storage-operator, ingress-operator, konnectivity-agent, redhat-operators-catalog, openshift-apiserver, cluster-image-registry-operator, redhat-marketplace-catalog, certified-operators-catalog, openshift-controller-manager, cluster-network-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"50cf928a4b9ab91ad463923b6c69c34145bea77427cbfe46d31360d594882c1c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"17cbac65ad6dde679afa9f8374f5b421e45ae642715ea520e35f5231ee1e4b90","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012312e0\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-mm9t5htdp974\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e464b7e0-c213-4e6e-988f-74f58142aa41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e464b7e0-c213-4e6e-988f-74f58142aa41\",\n      \"name\": \"e464b7e0-c213-4e6e-988f-74f58142aa41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:05:47.205015937Z\",\n      \"endTime\": \"2026-08-21T02:24:52.289709904Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, packageserver, redhat-marketplace-catalog, openshift-controller-manager, ingress-operator, olm-operator, redhat-operators-catalog, certified-operators-catalog, cluster-storage-operator, csi-snapshot-controller-operator, cluster-node-tuning-operator, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-policy-controller, cluster-network-operator, catalog-operator, community-operators-catalog, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012312e0\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-mm9t5htdp974\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e464b7e0-c213-4e6e-988f-74f58142aa41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e464b7e0-c213-4e6e-988f-74f58142aa41\",\n      \"name\": \"e464b7e0-c213-4e6e-988f-74f58142aa41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:05:47.205015937Z\",\n      \"endTime\": \"2026-08-21T02:24:52.289709904Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, packageserver, redhat-marketplace-catalog, openshift-controller-manager, ingress-operator, olm-operator, redhat-operators-catalog, certified-operators-catalog, cluster-storage-operator, csi-snapshot-controller-operator, cluster-node-tuning-operator, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-policy-controller, cluster-network-operator, catalog-operator, community-operators-catalog, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"f4ef26d460058d90ed4ee949e2b53c1556150d27ed9ab6fc4a76d0e97cfd95ed","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"fcefb659555e6a990fedc929fc65e2b32862b237e18ebc38834dc9a0eaf288eb","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, konnectivity-agent, openshift-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dba740\u003e: \n    failed to create HCP cluster ea-listfncws6: failed waiting for cluster=\"ea-listfncws6\" in resourcegroup=\"ea-list-zhgflbxgws4b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04c8d3ea-1f82-496d-aa9a-800028c0f1a6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04c8d3ea-1f82-496d-aa9a-800028c0f1a6\",\n      \"name\": \"04c8d3ea-1f82-496d-aa9a-800028c0f1a6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:35:24.937663979Z\",\n      \"endTime\": \"2026-08-21T02:54:33.431541939Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-policy-controller, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dba740\u003e: \n    failed to create HCP cluster ea-listfncws6: failed waiting for cluster=\"ea-listfncws6\" in resourcegroup=\"ea-list-zhgflbxgws4b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04c8d3ea-1f82-496d-aa9a-800028c0f1a6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04c8d3ea-1f82-496d-aa9a-800028c0f1a6\",\n      \"name\": \"04c8d3ea-1f82-496d-aa9a-800028c0f1a6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:35:24.937663979Z\",\n      \"endTime\": \"2026-08-21T02:54:33.431541939Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-policy-controller, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"5b3802d878773f1a0822d4136f1c861b8287fa1e3f87823166eff58535d9f1f9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"f969ddc66388f6636eb6fb562e07c82ce37d6c1699beba95e7c983471e1cd1bf","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, kube-controller-manager, kube-scheduler, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0028eeae0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-92dczdzgqxgg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/739a7635-b32c-4f38-8193-2975abc5e092\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/739a7635-b32c-4f38-8193-2975abc5e092\",\n      \"name\": \"739a7635-b32c-4f38-8193-2975abc5e092\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:40:33.871209807Z\",\n      \"endTime\": \"2026-08-21T02:59:33.678476291Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, kube-scheduler; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0028eeae0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-92dczdzgqxgg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/739a7635-b32c-4f38-8193-2975abc5e092\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/739a7635-b32c-4f38-8193-2975abc5e092\",\n      \"name\": \"739a7635-b32c-4f38-8193-2975abc5e092\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:40:33.871209807Z\",\n      \"endTime\": \"2026-08-21T02:59:33.678476291Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, kube-scheduler; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"01d3c79c792237fb62672cad293b186ea02a3ea2191a33420c6fdd434b27d6f3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"84c9b6f5f7cc00976416d8874ac20f25a776a6f9140090905fa5d8c5a508047b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001344800\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-q78wj4tcrbbm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31dc1e1d-a696-4310-84a7-03cd3b7f433b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31dc1e1d-a696-4310-84a7-03cd3b7f433b\",\n      \"name\": \"31dc1e1d-a696-4310-84a7-03cd3b7f433b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:44:35.391627616Z\",\n      \"endTime\": \"2026-08-21T03:03:43.616320763Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001344800\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-q78wj4tcrbbm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31dc1e1d-a696-4310-84a7-03cd3b7f433b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31dc1e1d-a696-4310-84a7-03cd3b7f433b\",\n      \"name\": \"31dc1e1d-a696-4310-84a7-03cd3b7f433b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:44:35.391627616Z\",\n      \"endTime\": \"2026-08-21T03:03:43.616320763Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"8239e5a1956b5200678e34e906157e9a5f4c31e6442b526e2f03f3fb5593beb2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"96ac8f4694917334827139385bf0e43172c70aa0ffc95a888499b14a4f14ff9d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [konnectivity-agent, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001178800\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-njrg4blgvtnt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5481864c-7cad-43c3-99c6-7f83d731f12b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5481864c-7cad-43c3-99c6-7f83d731f12b\",\n      \"name\": \"5481864c-7cad-43c3-99c6-7f83d731f12b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:29.121350695Z\",\n      \"endTime\": \"2026-08-21T02:22:32.49197336Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [konnectivity-agent deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001178800\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-njrg4blgvtnt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5481864c-7cad-43c3-99c6-7f83d731f12b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5481864c-7cad-43c3-99c6-7f83d731f12b\",\n      \"name\": \"5481864c-7cad-43c3-99c6-7f83d731f12b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:29.121350695Z\",\n      \"endTime\": \"2026-08-21T02:22:32.49197336Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [konnectivity-agent deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"dc7a3b65992904f6bd95eec51bb94c8945852f4be0b05c741ee7f46e3bc1a620","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"a4c7965c412b351e80ef0544bff250a8a694f957336d67b40d995be9b67f72d9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":3,"also_in":["DEV"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-stz97h\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005b4620\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-stz97h: failed waiting for cluster=\"cluster-zstream-4-21-stz97h\" in resourcegroup=\"rg-zstream-upgrade-4-21-jr9rvssp6jdl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\",\n      \"name\": \"dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:32:46.784949272Z\",\n      \"endTime\": \"2026-08-21T02:51:53.64362497Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, certified-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, ingress-operator, cluster-version-operator, cluster-policy-controller, dns-operator, cluster-image-registry-operator, csi-snapshot-controller-operator, catalog-operator, packageserver, olm-operator, redhat-operators-catalog, hosted-cluster-config-operator, cluster-node-tuning-operator, kube-controller-manager, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-stz97h\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005b4620\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-stz97h: failed waiting for cluster=\"cluster-zstream-4-21-stz97h\" in resourcegroup=\"rg-zstream-upgrade-4-21-jr9rvssp6jdl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\",\n      \"name\": \"dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:32:46.784949272Z\",\n      \"endTime\": \"2026-08-21T02:51:53.64362497Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, certified-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, ingress-operator, cluster-version-operator, cluster-policy-controller, dns-operator, cluster-image-registry-operator, csi-snapshot-controller-operator, catalog-operator, packageserver, olm-operator, redhat-operators-catalog, hosted-cluster-config-operator, cluster-node-tuning-operator, kube-controller-manager, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"8bf1efa5a500673bb5d80c68b86c9d236a42c04862fb84b7d85434272615d259","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"c9f88b4bf47ba7b1ded7ca8c11c50ed78accd7528c6672b7fec47ef01c0992a8","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T13:13:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010f8e20\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9qvbx4pls9rr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/668bdee1-fe77-422b-b467-4884f4f94057\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/668bdee1-fe77-422b-b467-4884f4f94057\",\n      \"name\": \"668bdee1-fe77-422b-b467-4884f4f94057\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T13:34:19.958633732Z\",\n      \"endTime\": \"2026-08-19T13:53:23.640928655Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010f8e20\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9qvbx4pls9rr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/668bdee1-fe77-422b-b467-4884f4f94057\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/668bdee1-fe77-422b-b467-4884f4f94057\",\n      \"name\": \"668bdee1-fe77-422b-b467-4884f4f94057\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T13:34:19.958633732Z\",\n      \"endTime\": \"2026-08-19T13:53:23.640928655Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090064577516539904","occurred_at":"2026-08-19T13:13:21Z","signature_id":"fdd6fe2f501aa81c3def2e90227b41f195f175a3dc5f2d2da6aeb9b38455a514","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"80ca3cd678a9c2df1da684df660807a18dfa93c6de640958f03cc3d04dad8c8d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T13:13:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e9e8e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wlkvs7868jnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e159d376-391e-45cb-a50d-4792a4a230e8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e159d376-391e-45cb-a50d-4792a4a230e8\",\n      \"name\": \"e159d376-391e-45cb-a50d-4792a4a230e8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T13:32:05.123641839Z\",\n      \"endTime\": \"2026-08-19T13:51:13.689263263Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: hosted-cluster-config-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e9e8e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wlkvs7868jnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e159d376-391e-45cb-a50d-4792a4a230e8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e159d376-391e-45cb-a50d-4792a4a230e8\",\n      \"name\": \"e159d376-391e-45cb-a50d-4792a4a230e8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T13:32:05.123641839Z\",\n      \"endTime\": \"2026-08-19T13:51:13.689263263Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: hosted-cluster-config-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090064577516539904","occurred_at":"2026-08-19T13:13:21Z","signature_id":"8fbb737fc21d5c791adf60078bc5daf0fea0e2cf41a6f52ad2aec57ec1f6dc65","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6722502d5d665731952f221ae4eff4c1fe60411f08b65ee3c8c1ea7fe37c4a2f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T12:36:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e68800\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rddwcsq6xrbg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf0b0684-dd10-4cf3-87c3-baf48258b7ce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf0b0684-dd10-4cf3-87c3-baf48258b7ce\",\n      \"name\": \"bf0b0684-dd10-4cf3-87c3-baf48258b7ce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:16:38.793143062Z\",\n      \"endTime\": \"2026-08-20T13:35:40.482871637Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e68800\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rddwcsq6xrbg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf0b0684-dd10-4cf3-87c3-baf48258b7ce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf0b0684-dd10-4cf3-87c3-baf48258b7ce\",\n      \"name\": \"bf0b0684-dd10-4cf3-87c3-baf48258b7ce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:16:38.793143062Z\",\n      \"endTime\": \"2026-08-20T13:35:40.482871637Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"37da2bd93766d6d4d4f6961c9e8500becdfb67c804e717112af85f0368264347","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"c6b1360d9b766b9ab9834bb704955b4878d98ef788b91a435d90ca99b95a962b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, machine-approver, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T13:13:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000806660\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-2c57lt79tp2l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e60c22e9-c32c-4219-b8d7-30131e20f6f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e60c22e9-c32c-4219-b8d7-30131e20f6f1\",\n      \"name\": \"e60c22e9-c32c-4219-b8d7-30131e20f6f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T13:33:35.958939424Z\",\n      \"endTime\": \"2026-08-19T13:52:43.668779021Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.m3b8f2l5y3g1d7d.320w.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000806660\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-2c57lt79tp2l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e60c22e9-c32c-4219-b8d7-30131e20f6f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e60c22e9-c32c-4219-b8d7-30131e20f6f1\",\n      \"name\": \"e60c22e9-c32c-4219-b8d7-30131e20f6f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T13:33:35.958939424Z\",\n      \"endTime\": \"2026-08-19T13:52:43.668779021Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.m3b8f2l5y3g1d7d.320w.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090064577516539904","occurred_at":"2026-08-19T13:13:21Z","signature_id":"d68e326b70076786b8e9b9f228d830ccd3f5c6fff7bafb198fa23e312a38b993","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"4b6bc70dde3370d5e666a11dcdf8b092a80443f417ead980036aeb5b45355b1f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, ignition-server, redhat-marketplace-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T08:35:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7af60\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-kfjt8dvwt56j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54f121a6-620a-4f93-bac1-3e84405aac12\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54f121a6-620a-4f93-bac1-3e84405aac12\",\n      \"name\": \"54f121a6-620a-4f93-bac1-3e84405aac12\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:44:43.862276988Z\",\n      \"endTime\": \"2026-08-20T10:03:52.490075448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.a8j1x1l8p8y5l8a.lecf.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7af60\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-kfjt8dvwt56j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54f121a6-620a-4f93-bac1-3e84405aac12\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54f121a6-620a-4f93-bac1-3e84405aac12\",\n      \"name\": \"54f121a6-620a-4f93-bac1-3e84405aac12\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:44:43.862276988Z\",\n      \"endTime\": \"2026-08-20T10:03:52.490075448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.a8j1x1l8p8y5l8a.lecf.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"84a396fab7c2ab3ae1588023933d5e474ab7b143128d353980cd15000536f48b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"dc30c2cd1479d681b3c8e85dbe87f378840f2bf26b28373ee01d7b48d759e41c","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server, konnectivity-agent, kube-scheduler]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-5lt48h with version 4.21.29\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001068a60\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-5lt48h: failed waiting for cluster=\"np-version-upgrade-cluster-5lt48h\" in resourcegroup=\"rg-np-version-upgrade-5lt48h-dszsrsbrq6zk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe765ae8-0a48-4400-a1c0-66c693e2b5fa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe765ae8-0a48-4400-a1c0-66c693e2b5fa\",\n      \"name\": \"fe765ae8-0a48-4400-a1c0-66c693e2b5fa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:45:36.662949608Z\",\n      \"endTime\": \"2026-08-21T03:04:44.745401761Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.q5e3m9a0x8h7s0u.7eq2.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-5lt48h with version 4.21.29\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001068a60\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-5lt48h: failed waiting for cluster=\"np-version-upgrade-cluster-5lt48h\" in resourcegroup=\"rg-np-version-upgrade-5lt48h-dszsrsbrq6zk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe765ae8-0a48-4400-a1c0-66c693e2b5fa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe765ae8-0a48-4400-a1c0-66c693e2b5fa\",\n      \"name\": \"fe765ae8-0a48-4400-a1c0-66c693e2b5fa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:45:36.662949608Z\",\n      \"endTime\": \"2026-08-21T03:04:44.745401761Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.q5e3m9a0x8h7s0u.7eq2.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"ff8b79ab35f4ee1da283d000f55c6cb2c5767888f63ecfe86c10a751852965b5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"c6e3b20deb9bae580ddd8091aa6a577e844f6b83692c9312152315f89fed55d3","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, cluster-api]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ce5680\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-s6nq8d7z6854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68d2c327-4b78-47bd-bbdb-95c6f476f1ab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68d2c327-4b78-47bd-bbdb-95c6f476f1ab\",\n      \"name\": \"68d2c327-4b78-47bd-bbdb-95c6f476f1ab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:04:33.015430331Z\",\n      \"endTime\": \"2026-08-21T02:23:42.504347676Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ce5680\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-s6nq8d7z6854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68d2c327-4b78-47bd-bbdb-95c6f476f1ab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68d2c327-4b78-47bd-bbdb-95c6f476f1ab\",\n      \"name\": \"68d2c327-4b78-47bd-bbdb-95c6f476f1ab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:04:33.015430331Z\",\n      \"endTime\": \"2026-08-21T02:23:42.504347676Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"60abba32a30fc314bf3f1591fe14612458a80c23ca459244b846d1098cad1d5d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"c94f79fadf5f0c5900ad5aba81751610a3b4c431f48765be0c4ce0b0f37bd7e9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001048e40\u003e: \n    failed to create HCP cluster np-dg-minor-dc29s7: failed waiting for cluster=\"np-dg-minor-dc29s7\" in resourcegroup=\"rg-np-dg-minor-dc29s7-9gmklftjh28x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\",\n      \"name\": \"6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:49:33.691417488Z\",\n      \"endTime\": \"2026-08-21T03:08:34.727253217Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001048e40\u003e: \n    failed to create HCP cluster np-dg-minor-dc29s7: failed waiting for cluster=\"np-dg-minor-dc29s7\" in resourcegroup=\"rg-np-dg-minor-dc29s7-9gmklftjh28x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\",\n      \"name\": \"6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:49:33.691417488Z\",\n      \"endTime\": \"2026-08-21T03:08:34.727253217Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"7ddd8e0bddb95c4a8ef63e85e6c7889f6f625f289311cf197d6695415a140810","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"db86d0a7b49ef93daf7cdb63d12e83b79ddcd71210e2847a436b8d80dd5dde7f","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] \u003cno_message\u003e; [clusterServiceStatus] \u003cno_message\u003e; [descendantResources] \u003cno_message\u003e; [hostedCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001716270\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-nml9dcvzzfr9\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n      \"name\": \"d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:14:11.789973414Z\",\n      \"endTime\": \"2026-08-24T23:38:15.678890881Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001716270\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-nml9dcvzzfr9\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n      \"name\": \"d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:14:11.789973414Z\",\n      \"endTime\": \"2026-08-24T23:38:15.678890881Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"3feed0e49b703d23a4823f602dec7fe62c46bf78411b06f90b5ee2c2bde72da5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"81be2fb29cef27cb213a54edbaf774e087ffc3ae2e0885f48e9fd38eeccf1e0f","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: externalAuths, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["DEV"],"trend_counts":[1,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-19T13:13:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a4a120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-drrz4q8s78wp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ae99f150-7046-4373-b424-e51c79b56a22\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ae99f150-7046-4373-b424-e51c79b56a22\",\n      \"name\": \"ae99f150-7046-4373-b424-e51c79b56a22\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T14:30:35.01030213Z\",\n      \"endTime\": \"2026-08-19T14:54:35.627231464Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9mlei3q3ci7sc5kdghjha0ako2p2s2 still exists (deletion dispatched at 2026-08-19T14:30:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a4a120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-drrz4q8s78wp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ae99f150-7046-4373-b424-e51c79b56a22\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ae99f150-7046-4373-b424-e51c79b56a22\",\n      \"name\": \"ae99f150-7046-4373-b424-e51c79b56a22\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T14:30:35.01030213Z\",\n      \"endTime\": \"2026-08-19T14:54:35.627231464Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9mlei3q3ci7sc5kdghjha0ako2p2s2 still exists (deletion dispatched at 2026-08-19T14:30:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090064577516539904","occurred_at":"2026-08-19T13:13:21Z","signature_id":"e904a0dd0e231e642877fcf061d837be87b8253a6f6c4f6c048159da596c5ace","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"ee72f6e8217d23c4c37addc0dee5930c5b90fe95a9029df3ed16ce9a3769fd8f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-17-010411 to 4.23.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T23:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:174]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-w8t8gk\" to 4.23.0-0.nightly-multi-2026-08-19-140455\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00160f3a0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-w8t8gk\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-ldgb8b99w4xp\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/97cf82cb-bb1f-4aac-aaea-96415f8aa42a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/97cf82cb-bb1f-4aac-aaea-96415f8aa42a\",\n      \"name\": \"97cf82cb-bb1f-4aac-aaea-96415f8aa42a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:45:11.57498354Z\",\n      \"endTime\": \"2026-08-19T23:45:21.100010011Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-17-010411 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:174]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-w8t8gk\" to 4.23.0-0.nightly-multi-2026-08-19-140455\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00160f3a0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-w8t8gk\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-ldgb8b99w4xp\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/97cf82cb-bb1f-4aac-aaea-96415f8aa42a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/97cf82cb-bb1f-4aac-aaea-96415f8aa42a\",\n      \"name\": \"97cf82cb-bb1f-4aac-aaea-96415f8aa42a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:45:11.57498354Z\",\n      \"endTime\": \"2026-08-19T23:45:21.100010011Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-17-010411 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"337a8761f22e379b1943e8bf13e36bffe4d94278c9dc5ca3fef8e2f8362ca41d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"8dad9a32124d40126133bba2520cede07e7f2a6cd0df33b90c1e690aeb9f3131","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.20\": must be at least 4.20.34; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T02:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-4lxbwp\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011b1e00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-p9w9mgdh26kn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-4lxbwp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-4lxbwp\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011b1e00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-p9w9mgdh26kn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-4lxbwp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","signature_id":"4ebee03d5cb8560c61e868d8bbb831f73868f90ba39c7b26a29c917e8892dc28","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7ce22653c1790596420acf5ec30dfcb3f90e95095df0cfef47397e47b57f8997","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21\": must be at least 4.21.29; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T02:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-qkbdvr\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008bcae0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qdqdx9dwcprj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-qkbdvr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-qkbdvr\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008bcae0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qdqdx9dwcprj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-qkbdvr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","signature_id":"8e749cbda0987bd36e3427fb17c74e6efb4a7ffd7f85d804ab73f254f7c2b973","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"4a4f87d4133fd3a9df163a91b47668459c543343ba1d123ba27a73ceaa3385ce","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.22\": must be at least 4.22.9; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T02:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-fnbgn2\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011309f0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-cbhdqjqgzmjz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-fnbgn2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-fnbgn2\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011309f0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-cbhdqjqgzmjz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-fnbgn2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","signature_id":"2e88e71b6a16994e8d9342a6dfffd6392efeb37bdd6569b14205b6f91d0d4a24","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"0b5ec0cbb59802a88f2bc42deba73fa247a353c118d64efe8e80ca88b66f3f5b","failure_pattern":"Test Panicked","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475]: Test Panicked","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-22T02:00:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["github.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0xc001716000?, {0x7f5ef448fe78?, 0xc000f7dd10?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:1039 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0xc000aceb00, {0x7f5ef448fe78, 0xc000f7dd10}, {0xc001436078, 0x17}, {0xc0000140b9, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:464 +0x2a9\ngithub.com/Azure/ARO-HCP/test/e2e.init.func26.1({0x7f5ef448fe78, 0xc000f7dd10}, {0xeb544f6, 0x4})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:88 +0x6a5\nreflect.Value.call({0xc477720?, 0xf1650a0?, 0x13?}, {0xeb54646, 0x4}, {0xc000f7dda0, 0x2, 0x2?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcca\nreflect.Value.Call({0xc477720?, 0xf1650a0?, 0xfc03900?}, {0xc000f7dda0?, 0x20?, 0xc755ac0?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9\ngithub.com/onsi/ginkgo/v2.invokeFunction({0xc477720, 0xf1650a0}, {0xc000f5d6e0, 0x2, 0xc755ac0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:339 +0x2c5\ngithub.com/onsi/ginkgo/v2.generateTable.func2.1({0xfb6b1a0?, 0xc000f7dd10?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 101\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475]: Test Panicked: failed to get subscription ID: subscription with name 'XXXXXXXXXXX Hosted Clusters - Stage - 00' not found\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475]: Test Panicked\ngithub.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0xc001716000?, {0x7f5ef448fe78?, 0xc000f7dd10?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:1039 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0xc000aceb00, {0x7f5ef448fe78, 0xc000f7dd10}, {0xc001436078, 0x17}, {0xc0000140b9, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:464 +0x2a9\ngithub.com/Azure/ARO-HCP/test/e2e.init.func26.1({0x7f5ef448fe78, 0xc000f7dd10}, {0xeb544f6, 0x4})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:88 +0x6a5\nreflect.Value.call({0xc477720?, 0xf1650a0?, 0x13?}, {0xeb54646, 0x4}, {0xc000f7dda0, 0x2, 0x2?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcca\nreflect.Value.Call({0xc477720?, 0xf1650a0?, 0xfc03900?}, {0xc000f7dda0?, 0x20?, 0xc755ac0?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9\ngithub.com/onsi/ginkgo/v2.invokeFunction({0xc477720, 0xf1650a0}, {0xc000f5d6e0, 0x2, 0xc755ac0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:339 +0x2c5\ngithub.com/onsi/ginkgo/v2.generateTable.func2.1({0xfb6b1a0?, 0xc000f7dd10?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 101\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475]: Test Panicked: failed to get subscription ID: subscription with name 'XXXXXXXXXXX Hosted Clusters - Stage - 00' not found\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475]: Test Panicked"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"8a15fba718b6d75d8d44dad039889bf9f5ba45b37ef0458c98c5fa51b3f5ce6d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b748a3b1c1b9c7fd8f7f617ddf190f4c65999a469c2f67349a1718ac3211e788","failure_pattern":"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5","search_query":"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0013e8a98\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-08-20-024947, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: failed to get clusterversion \"version\": Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": EOF\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/version\": EOF\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": EOF\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0013e8a98\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-08-20-024947, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: failed to get clusterversion \"version\": Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": EOF\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/version\": EOF\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": EOF\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"9597e11d65e1b6b9e701068b23031343892ee3c2b207fcc034b59895dd98d0c1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"bd4fdaffaf3b15f286ad5f2ca30cb95f7541a63bca5896bf32cc0ecd46a82996","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"\u003curl\u003e","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000fcc5b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-20-065923, targetMinor=4.22.0-0.nightly-multi-2026-08-20-141954) failed: failed to get clusterversion \"version\": Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000fcc5b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-20-065923, targetMinor=4.22.0-0.nightly-multi-2026-08-20-141954) failed: failed to get clusterversion \"version\": Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"28d3b5e4d6e1e3b6c0885d78d546ff6d719fb6e7e972de708efbba62c246ae05","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3da6dfe5e2d468b0f7647b604914d8743369fe4db63b9b1f8f365f64663404b8","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":16,"also_in":["DEV","PROD"],"trend_counts":[1,0,0,1,0,0,0,0],"trend_range":"2026-08-18..2026-08-25 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-10","2026-08-17"],"prior_runs_affected":17,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"d06b61b2a290f31b2b44796608e3a50ed66dd83e5cff65620f4a4dbd778567f6","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}