{"meta":{"start_date":"2026-09-23","end_date":"2026-09-23","start_at":"2026-09-23T00:00:00Z","end_at":"2026-09-24T00:00:00Z","timezone":"UTC","generated_at":"2026-10-14T11:02:57Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":97,"failed_runs":69,"raw_occurrences":143,"matched_occurrences":138,"runs_affected":64},"rows":[{"environment":"dev","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":34,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.0618556701030926,"anchor_occurrences":34,"after_last_push_count":78,"trend_counts":[26,0,0,0,29,56,34],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":6,"prior_last_seen_at":"2026-09-17T00:56:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--8w2v2h7tbr7x/cluster-hs-pre-9brzc9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000854de0\u003e: \n failed to create HCP cluster cluster-hs-pre-9brzc9: failed waiting for cluster=\"cluster-hs-pre-9brzc9\" in resourcegroup=\"rg-hs-pre--8w2v2h7tbr7x\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6abdd772-bc76-4739-9454-989e477a880a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6abdd772-bc76-4739-9454-989e477a880a\",\n \"name\": \"6abdd772-bc76-4739-9454-989e477a880a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T22:56:17.609215386Z\",\n \"endTime\": \"2026-09-23T23:25:22.953425865Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--8w2v2h7tbr7x/cluster-hs-pre-9brzc9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000854de0\u003e: \n failed to create HCP cluster cluster-hs-pre-9brzc9: failed waiting for cluster=\"cluster-hs-pre-9brzc9\" in resourcegroup=\"rg-hs-pre--8w2v2h7tbr7x\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6abdd772-bc76-4739-9454-989e477a880a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6abdd772-bc76-4739-9454-989e477a880a\",\n \"name\": \"6abdd772-bc76-4739-9454-989e477a880a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T22:56:17.609215386Z\",\n \"endTime\": \"2026-09-23T23:25:22.953425865Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-96r455f6cgr8/cluster-candidate-4-21-tc5xmb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ba4aa0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-tc5xmb: failed waiting for cluster=\"cluster-candidate-4-21-tc5xmb\" in resourcegroup=\"rg-candidate-4-21-96r455f6cgr8\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2add970-69a8-4470-839c-381250f38990\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2add970-69a8-4470-839c-381250f38990\",\n \"name\": \"c2add970-69a8-4470-839c-381250f38990\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T22:56:17.620493086Z\",\n \"endTime\": \"2026-09-23T23:25:22.742938899Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-96r455f6cgr8/cluster-candidate-4-21-tc5xmb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ba4aa0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-tc5xmb: failed waiting for cluster=\"cluster-candidate-4-21-tc5xmb\" in resourcegroup=\"rg-candidate-4-21-96r455f6cgr8\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2add970-69a8-4470-839c-381250f38990\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2add970-69a8-4470-839c-381250f38990\",\n \"name\": \"c2add970-69a8-4470-839c-381250f38990\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T22:56:17.620493086Z\",\n \"endTime\": \"2026-09-23T23:25:22.742938899Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bf1200\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-dx59g4bq6jrp\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a97c4cb5-238d-43ff-b943-3f685b71b6f1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a97c4cb5-238d-43ff-b943-3f685b71b6f1\",\n \"name\": \"a97c4cb5-238d-43ff-b943-3f685b71b6f1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T22:56:16.987949919Z\",\n \"endTime\": \"2026-09-23T23:25:22.924061821Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bf1200\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-dx59g4bq6jrp\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a97c4cb5-238d-43ff-b943-3f685b71b6f1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a97c4cb5-238d-43ff-b943-3f685b71b6f1\",\n \"name\": \"a97c4cb5-238d-43ff-b943-3f685b71b6f1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T22:56:16.987949919Z\",\n \"endTime\": \"2026-09-23T23:25:22.924061821Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"0a6194beb78b8c406f21d0c39e8f4938c36409cacbef3176434d6c29c859c82b","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"0c65583f9cf2766f075edef8345e43c644c0d3a9639fd26af02b95e481bc76ec","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"4342f7d49a31fc62a7cd4188188d768f7c7622da6926b4fcd1d52c2f7741506a","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"7fc420454baa557ad64e773af2ec483367dc179f35828ca014109dbc19bd2053","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"813c3b4a34551005c00d10be781d74a14c7bbb31e37f764d60afab74ae06652e","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"9eb53b927bd56443c24b7b20ca5bab6adea62beccaa9c3e083e3ba49ff15f1b7","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"a08dcc436350e9ea389eb3d5efb0662df729d203d30b15f5920616ae5fc224ca","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"c525979fb2d08507f3d446b04e74f3e5b602e0906716c705201cd81002770ba7","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"cb11f1de63b86677e85bb10ffef4875807561668b3123fc194016353354c7ac8","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"d8dcd8028ce1adacc03f46fb0158f289f63ae31f72e8e4ce40b790b255cba87a","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"ef425b9fc641abb8b932709dc29ce158807de6ae50f47a33b3db89e10d6a6b93","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"0adb4bb4ae9202e459b2315ccfce11f495585f3d7bac3fc7a2487d9796add3c7","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"1b19c6dab2da849dbdc19f25d8a7c8a64dadf6697567bfe5d10096294dd0a6e4","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"2df52c1cafa5532c454b36cb7b084b68593cebfadd2864d45dd1a118200b8d52","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"2f81637f8265c258ea75aa24f9fd9432ef36114b9ebee46504be7d5770c435b7","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"36a05e474622bc3fcc801d228958fffc4a6d075602e55266d09e5b2e91c19368","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"381000ecc9d6086c37b0219505b21d93a9c939a2ccf23f03576b38ab84b5a55e","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"449030cbb33da5d4c8dac990dceda44dd62ccc5ca1ffc1ccd12391d7b55e0681","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"47903548f1f143d7ab9998fc3c8c1fbc469323e86e2d9a0d0fddfcc0a731af5d","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"51f8c6265949434e6442ba80f7ade4730dcad12175f31515ce73c25444d377e5","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"5f3ee90b05be0d7f1487912ea92f8d3f0441c3abfa713a796fa8f25021c270ac","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"6fc513af79b2809dcb5d5df1bdafeac7cf5d4c3a1862412ed3eba47ccae158c5","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"89edcec5785330a2ee63b672917ff746b2f687ab7ded705d0cc185c95d8bec14","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"9731f91656941bbb9380fa37cca11d45aa6ad2c56b1236e28cb4401092e4c0e4","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"97bb982d1e6ea1aa19045a58c60ec413fe3ba3d8834e3141a02e5782483f6785","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"9e823b348b35d763f4f7278b55c97380508738106688d0cd3c61c152c24157ec","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"a465030c91e7749c08c2c7314b00f14fac042b380238eb18631f6fc0cc7ad978","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"b4f75fac021e402acc2db264f349c96a35364d535bee0b2e0065ea599920a92f","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"cb7663c2933e4c98c3163332183af2e5279eff6073b28c84e67ab60d8693fc1a","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"cfc810c40e60a646985d0ea21bde9b0fc719a1179e342cbce3fe6eb38a62ed69","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"d6cea3ffddfab34ef89656bb4911dcfd537e070bc053f39a0b804e5db90f21f8","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"dd70726fbf025a2e22838d8e9a9bcfaa30032d7ec07810dbb606326dc89fe874","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"e1d6b4b1a86d1907639ae72cc2ccc8137426d24a60ad5b4f6f3e7247ee41a9cc","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"fb3c31a801804fd94e8c51eb7bcff2f2193b26a3f6b7ab9373ac03f4a1eac932","pr_number":7108,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":22,"runs_affected":22,"failed_runs":22,"run_impact_percent":22.68041237113402,"anchor_occurrences":22,"after_last_push_count":46,"trend_counts":[7,11,6,5,20,21,22],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":100,"prior_last_seen_at":"2026-09-20T23:13:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":22}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T21:52:45Z\n Ended: 2026-09-23T22:01:44Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4706560-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4706560-svc).","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T20:59:04Z\n Ended: 2026-09-23T21:31:03Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9340032-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9340032-svc).","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T19:49:29Z\n Ended: 2026-09-23T20:23:30Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4862464-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4862464-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102855242784706560","occurred_at":"2026-09-23T20:18:53Z","signature_id":"23e9b2995fc9ded2e9caf55c0eceabc998d92c6176e04363cf44c1eee0ec317d","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","occurred_at":"2026-09-23T19:52:03Z","signature_id":"9768ce66171b43df472ff44180bc3d8e585fee001b4346d3ab9374574a7b151f","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102829281724862464","occurred_at":"2026-09-23T18:35:44Z","signature_id":"766b1d46f5072f7c906c6126dd0836e5f198da05367c355a94ce2d777ad7547e","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","occurred_at":"2026-09-23T17:26:00Z","signature_id":"56b2ff341978727c646e219e23fba2dfc1ef2e5ccc4828702e0dece8d2810ba6","pr_number":7117,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800084281331712","occurred_at":"2026-09-23T16:39:42Z","signature_id":"aca35d9e741dfeba834ff114bffafa2a455703b5a32f415b37bdcbd261eb6a67","pr_number":7090,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102769175150727168","occurred_at":"2026-09-23T14:36:53Z","signature_id":"3135ce76bbdd1ef05af982db7996ed6329db94eadabfb94067b89a304dc4d2cb","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102759065523326976","occurred_at":"2026-09-23T13:56:43Z","signature_id":"b522a7cea6c2812e29c1ea3b0132f66a43bf87236c4f0123b2bd527a73ab14bb","pr_number":7054,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102744695867183104","occurred_at":"2026-09-23T12:59:37Z","signature_id":"f10cdf2980f54cb7434c563cea5c00835d86c81de376364c86add2439c33d20f","pr_number":7105,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","occurred_at":"2026-09-23T12:35:31Z","signature_id":"e8f9ea179e852213d3a81d66ef232d1e14041b2de7883a9afe6b09a4073f601b","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7103/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102724367396900864","occurred_at":"2026-09-23T11:38:49Z","signature_id":"66ef3a438e94ca88c6470f62824d14213734daf6ab2142316c1d734ddeaa9c0e","pr_number":7103,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102719452461142016","occurred_at":"2026-09-23T11:19:17Z","signature_id":"04cf9b464f041c4aa5f1606316891b1c2238b1191be6f182872d69d9ca55ffea","pr_number":7072,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102716371392532480","occurred_at":"2026-09-23T11:07:04Z","signature_id":"ecc22c0637cf2c040e23438995345ce727a3526729f1b45629b38665f2e2a492","pr_number":6439,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102700766367059968","occurred_at":"2026-09-23T10:05:03Z","signature_id":"695f2b58900eeb6ddc3f5c28514b819f16afd04fc4ec5e5122696e6f1efc6a9c","pr_number":7065,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102692523162996736","occurred_at":"2026-09-23T09:32:18Z","signature_id":"5be4b79fd707f0f902f0c6190406a62281b77b6ad2601f0cc1b46e4e850c930c","pr_number":6226,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102692072241762304","occurred_at":"2026-09-23T09:30:30Z","signature_id":"a430843224ed1479b519c2f15a1c5fc4265a858e6f819daf935153b2c4d45bf9","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102688892556677120","occurred_at":"2026-09-23T09:17:52Z","signature_id":"168b46660d0ed6f4bf2e6d271e550940fff7e50d9ae0544d3d77fc3aaa3c674c","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102669953994854400","occurred_at":"2026-09-23T08:02:37Z","signature_id":"ef8ed7f81b6e444cb9f47b23d2f051a1e779787724507e05e79e426102d41cd3","pr_number":7072,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651461333684224","occurred_at":"2026-09-23T06:49:08Z","signature_id":"24389b82f6419bf8544e5612ab91556f9033a43c11c58c8a75f9eddd0d3cb5d2","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7099/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650881433407488","occurred_at":"2026-09-23T06:46:50Z","signature_id":"7c059f6930e4e70e01cf60cdc5a209a41e1d36e3c5cdc0a3889876307b50e19b","pr_number":7099,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650243035172864","occurred_at":"2026-09-23T06:44:17Z","signature_id":"9b7c4ba8bc11e40e7d6443d7bd87ac580a2605e35dba3d0581d0e43dcef62d3d","pr_number":7041,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102624604781547520","occurred_at":"2026-09-23T05:01:42Z","signature_id":"e61751d58352ab491f0598c208a923b5d0b7b5b03f412d966b5363851718b33b","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579128883482624","occurred_at":"2026-09-23T02:01:18Z","signature_id":"b52a16123591cbdade9825eddb9e37de34eaab046973c4b8e648b0ae88335846","pr_number":6922,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":18,"runs_affected":18,"failed_runs":18,"run_impact_percent":18.556701030927837,"anchor_occurrences":18,"after_last_push_count":34,"trend_counts":[2,1,3,3,17,31,18],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":79,"prior_last_seen_at":"2026-09-20T23:13:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":18}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T00:08:12Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j7507200-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-rpfwd\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-rpfwd (namespace arobit) on cluster ci01-j7507200-svc has averaged 126.1% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T22:48:48Z\n Ended: 2026-09-23T23:00:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1922944-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-869c9489f8-dbnnd\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-869c9489f8-dbnnd (namespace kube-applier) on cluster ci01-j1922944-mgmt-1 has averaged 124.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-23T23:16:44Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1922944-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-z6smt\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-z6smt (namespace arobit) on cluster ci01-j1922944-svc has averaged 124.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-23T23:11:40Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8888448-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-m4cdw\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-m4cdw (namespace arobit) on cluster ci01-j8888448-svc has averaged 124.2% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102883816627507200","occurred_at":"2026-09-23T22:12:26Z","signature_id":"45c5b936de9aef65cab348b5a9c37cd387b152250ed997df53a6e8778dabfb08","pr_number":7020,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102872411731922944","occurred_at":"2026-09-23T21:27:07Z","signature_id":"aef4de3c1d8c7e8e67123da5e644cdc4d14dd0dceea99369d4b941edf923b506","pr_number":7123,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102871802198888448","occurred_at":"2026-09-23T21:24:41Z","signature_id":"d71eb8d3e94b0946ce9d45150994a79c1da31f720176e1cb83633088dcca59fb","pr_number":7119,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102862951168872448","occurred_at":"2026-09-23T20:49:31Z","signature_id":"a051597c46d9f96e8b20b9310e059e4f799849ba3ec132c05941edb67268ea29","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102855242784706560","occurred_at":"2026-09-23T20:18:53Z","signature_id":"254dd39e2f0e4cec1b71ce202f610ab07b9da883c1b746d73e09e4f13421506a","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840297615527936","occurred_at":"2026-09-23T19:19:30Z","signature_id":"8d0ffdcc532373f83199e900f86943923bf9464de78da96a0bc81c5d10748030","pr_number":7054,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840286248964096","occurred_at":"2026-09-23T19:19:27Z","signature_id":"f01e8023db70ac4eb379611e2c78c4d54f1aeadca78f25f3826d0ffec54edaa2","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102829281724862464","occurred_at":"2026-09-23T18:35:44Z","signature_id":"312cea2b73363d1b160f9cce1d956a42a926ffe6d5a41922d416b5287eec53f4","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","occurred_at":"2026-09-23T17:26:00Z","signature_id":"cedc29dd595c13bb38b5f8171952695156bf1b2712e54cef49d283d640a3a08e","pr_number":7117,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800950539325440","occurred_at":"2026-09-23T16:43:09Z","signature_id":"f1b24a151cda52cd0d54ea6a4677f8fb34ef83c5bdadd527eb334c9d72e78bde","pr_number":6925,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102795261179858944","occurred_at":"2026-09-23T16:20:32Z","signature_id":"c0de1a0ccf1f7966d446e232f88e4d90f26f4017b261cdeb15d15f7eb77aa118","pr_number":7054,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102764619155116032","occurred_at":"2026-09-23T14:18:47Z","signature_id":"df2c8e2eea124fae55058fbdd964feff88c2979d18d038ec6b6702d5cd247db8","pr_number":6687,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102751455361372160","occurred_at":"2026-09-23T13:26:28Z","signature_id":"27fba5395347858a8e8e715d32f7f0fe022ff7065fe1ea71ae9a42e0ed9a0a0d","pr_number":7044,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","occurred_at":"2026-09-23T12:35:31Z","signature_id":"c5b74c40552ae1b36fab99026dc41a2a915f1d1a792b500821eca9610ee3d31e","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7103/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102724367396900864","occurred_at":"2026-09-23T11:38:49Z","signature_id":"b94154bec553ae4d7de90c235afb929dc5f8241123d4ff71acb4b324e3664bec","pr_number":7103,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102669943756558336","occurred_at":"2026-09-23T08:02:34Z","signature_id":"6a553ba910a4f3a893c5726596a17745a39160fc0a64f583a6cc5c05a5a4ea9e","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651461333684224","occurred_at":"2026-09-23T06:49:08Z","signature_id":"61fa061764ad73e8317dc73f0ca153b35f59fa073234e6998c92f96b627530c7","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650243035172864","occurred_at":"2026-09-23T06:44:17Z","signature_id":"398824eec9025702d224d5a9ba36e64c2703dfdc5e00ba0eb6afbc886a9d202d","pr_number":7041,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":7.216494845360825,"anchor_occurrences":7,"after_last_push_count":20,"also_in":["PROD"],"trend_counts":[6,4,1,5,3,8,7],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":38,"prior_last_seen_at":"2026-09-20T23:13:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a9f0c0\u003e: \n failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-94qwsncfglvx\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a2c4ba6-7b02-45b6-b246-27301c3c7b44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a2c4ba6-7b02-45b6-b246-27301c3c7b44\",\n \"name\": \"9a2c4ba6-7b02-45b6-b246-27301c3c7b44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T23:07:02.752849186Z\",\n \"endTime\": \"2026-09-23T23:26:11.541839688Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a9f0c0\u003e: \n failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-94qwsncfglvx\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a2c4ba6-7b02-45b6-b246-27301c3c7b44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a2c4ba6-7b02-45b6-b246-27301c3c7b44\",\n \"name\": \"9a2c4ba6-7b02-45b6-b246-27301c3c7b44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T23:07:02.752849186Z\",\n \"endTime\": \"2026-09-23T23:26:11.541839688Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004d5460\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-8fx9vqqhjrvz\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ca16a72-f387-4f26-a7d9-a173356a7d9b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ca16a72-f387-4f26-a7d9-a173356a7d9b\",\n \"name\": \"5ca16a72-f387-4f26-a7d9-a173356a7d9b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T20:52:19.857894326Z\",\n \"endTime\": \"2026-09-23T21:11:21.209971005Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004d5460\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-8fx9vqqhjrvz\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ca16a72-f387-4f26-a7d9-a173356a7d9b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ca16a72-f387-4f26-a7d9-a173356a7d9b\",\n \"name\": \"5ca16a72-f387-4f26-a7d9-a173356a7d9b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T20:52:19.857894326Z\",\n \"endTime\": \"2026-09-23T21:11:21.209971005Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006be420\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-4wt68pfrx42w\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/adb5be9c-18bf-41aa-925f-6eb547095403\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/adb5be9c-18bf-41aa-925f-6eb547095403\",\n \"name\": \"adb5be9c-18bf-41aa-925f-6eb547095403\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T18:46:32.650200261Z\",\n \"endTime\": \"2026-09-23T19:05:38.929512243Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006be420\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-4wt68pfrx42w\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/adb5be9c-18bf-41aa-925f-6eb547095403\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/adb5be9c-18bf-41aa-925f-6eb547095403\",\n \"name\": \"adb5be9c-18bf-41aa-925f-6eb547095403\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T18:46:32.650200261Z\",\n \"endTime\": \"2026-09-23T19:05:38.929512243Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102883816627507200","occurred_at":"2026-09-23T22:12:26Z","signature_id":"0c93d7531b51f1ea100516ccd01b0a1c6615ecf8a2bbdc599e9f78e9e71a81e6","pr_number":7020,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","occurred_at":"2026-09-23T20:01:44Z","signature_id":"88d7c43f7ba21a8263fb8b4c871ac116be74ea1e4b599156e118374c23dbf6bf","pr_number":7110,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102818882191364096","occurred_at":"2026-09-23T17:54:24Z","signature_id":"d95e88444841564bfebc7b4d86984dd9a38294a94ca9cf72a10f4adf98342abd","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7114/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800348405043200","occurred_at":"2026-09-23T16:40:45Z","signature_id":"8a478233ef9f4819c24a5dc34485444f654a99c79dc829ccb980b0bc2625e398","pr_number":7114,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7106/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102751439267827712","occurred_at":"2026-09-23T13:26:24Z","signature_id":"e4012e513d3cafa3f6c8980450c912f0d288e7cf2e0c893549e7a66042041837","pr_number":7106,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102750533910532096","occurred_at":"2026-09-23T13:22:24Z","signature_id":"8eb0ca7714bb43bcafee5fa2084b2284a94351ac2160b6d55e893d478c2e48d8","pr_number":6439,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","occurred_at":"2026-09-23T12:35:31Z","signature_id":"a9f1409cf050d25886bb76839830611031e806e434f180cb1affd1b0a192d798","pr_number":6993,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message The zone '\u003chcp-api-host\u003e' is not available.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":4.123711340206185,"anchor_occurrences":4,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":4}],"full_error_samples":["time=2026-09-23T23:25:15.442Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-23T23:25:17.399Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-23T23:25:18.602Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=72a5eef9024c0fb803ba8efe57568ddcaa2d06a84e61744f2eeaeba881052bfe portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9983616%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F72a5eef9024c0fb803ba8efe57568ddcaa2d06a84e61744f2eeaeba881052bfe\ntime=2026-09-23T23:27:49.191Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9983616/providers/Microsoft.Resources/deployments/72a5eef9024c0fb803ba8efe57568ddcaa2d06a84e61744f2eeaeba881052bfe/operationStatuses/08584114009676534307\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The zone 'aro-hcp-dev-shard1-slot-01.hcp.osadev.cloud' is not available.\\\\\\\"\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-23T22:31:07.038Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-23T22:31:08.814Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-23T22:31:10.117Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=cd29b4bde2b5b7602cb211d98c814688cd52f3aab95b368210e0a8cfb85babb1 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8206336%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcd29b4bde2b5b7602cb211d98c814688cd52f3aab95b368210e0a8cfb85babb1\ntime=2026-09-23T22:34:11.270Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8206336/providers/Microsoft.Resources/deployments/cd29b4bde2b5b7602cb211d98c814688cd52f3aab95b368210e0a8cfb85babb1/operationStatuses/08584114042161978286\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The zone 'aro-hcp-dev-shard1-slot-01.hcp.osadev.cloud' is not available.\\\\\\\"\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-23T21:07:35.163Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-23T21:07:36.837Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-23T21:07:38.387Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=5d60488de7a83700a82e6fe6e3ffcdc28baad600277f95b8571fc3d6f39546ee portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4221952%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F5d60488de7a83700a82e6fe6e3ffcdc28baad600277f95b8571fc3d6f39546ee\ntime=2026-09-23T21:10:39.591Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4221952/providers/Microsoft.Resources/deployments/5d60488de7a83700a82e6fe6e3ffcdc28baad600277f95b8571fc3d6f39546ee/operationStatuses/08584114092280138717\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The zone 'aro-hcp-dev-shard1-slot-01.hcp.osadev.cloud' is not available.\\\\\\\"\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102895347519983616","occurred_at":"2026-09-23T22:58:15Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881020008206336","occurred_at":"2026-09-23T22:01:19Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":7112,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102862954004221952","occurred_at":"2026-09-23T20:49:32Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":7115,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6733/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102854046258827264","occurred_at":"2026-09-23T20:14:08Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":6733,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":4.123711340206185,"anchor_occurrences":4,"after_last_push_count":6,"trend_counts":[1,4,1,1,3,6,4],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":23,"prior_last_seen_at":"2026-09-20T16:10:57Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":4}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T23:16:20Z\n Ended: 2026-09-23T23:41:14Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7507200-svc\", component=\"backend\", name=\"datadump\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue datadump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T21:06:19Z\n Ended: 2026-09-23T21:21:20Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9340032-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-09-23T21:06:19Z\n Ended: 2026-09-23T21:36:17Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9340032-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 3:\n State: Resolved\n Started: 2026-09-23T21:06:19Z\n Ended: 2026-09-23T21:36:17Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9340032-svc\", component=\"backend\", name=\"datadump\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue datadump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 4:\n State: Resolved\n Started: 2026-09-23T21:06:19Z\n Ended: 2026-09-23T21:21:20Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9340032-svc\", component=\"backend\", name=\"clusterclusterserviceupdatedispatch\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterclusterserviceupdatedispatch has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T16:33:04Z\n Ended: 2026-09-23T16:39:07Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4918656-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102883816627507200","occurred_at":"2026-09-23T22:12:26Z","signature_id":"125273ad41e8d32bfd551d1beb0bc45baf8569efc2fb05f6f2496e0c967d64ac","pr_number":7020,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","occurred_at":"2026-09-23T19:52:03Z","signature_id":"71f4c03d6bb4614602efd1761f8bbf73a703154b3346a44390285f109ca0b681","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102782984254918656","occurred_at":"2026-09-23T15:31:45Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579128883482624","occurred_at":"2026-09-23T02:01:18Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":6922,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.0927835051546393,"anchor_occurrences":3,"after_last_push_count":17,"trend_counts":[0,3,3,5,4,13,3],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":31,"prior_last_seen_at":"2026-09-20T19:55:27Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T21:26:02Z\n Ended: 2026-09-23T21:34:02Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j4857344-svc\", component=\"frontend\", method=\"post\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential\", severity=\"warning\"\n Description: More than 5% of frontend requests for post /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential exceeded 1 second over the past 30 minutes (cluster ci01-j4857344-svc).","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T18:53:46Z\n Ended: 2026-09-23T19:08:44Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j5312512-svc\", component=\"frontend\", method=\"delete\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j5312512-svc).","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T03:37:03Z\n Ended: 2026-09-23T03:47:04Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j3482624-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j3482624-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","occurred_at":"2026-09-23T20:01:44Z","signature_id":"d5d990830365516d1467804cae51c493981b29d7c2296cdf7adf893366ce4712","pr_number":7110,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102812400645312512","occurred_at":"2026-09-23T17:28:39Z","signature_id":"c1db43e1f9ed556a1e717aa9ea521e513132026799cb40e29c7cc1976942dff0","pr_number":7115,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579128883482624","occurred_at":"2026-09-23T02:01:18Z","signature_id":"b20cadb846b55234efc43634cdf321fda9be13c6186765f74dd9bcf81c873e27","pr_number":6922,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.0618556701030926,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[1,1,0,0,1,2,2],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":9,"prior_last_seen_at":"2026-09-18T11:54:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":7108,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.0618556701030926,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":6,"prior_last_seen_at":"2026-09-16T15:56:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f1c4c0\u003e: \n DNS for route host agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f1c4c0\u003e: \n DNS for route host agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-pt8mvm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00118c460\u003e: \n DNS for route host agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-pt8mvm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00118c460\u003e: \n DNS for route host agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102782984254918656","occurred_at":"2026-09-23T15:31:45Z","signature_id":"0c29798c8c1659484795e61241cd7388e309cd731fe979947b39b617cfeb82bb","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102762797665357824","occurred_at":"2026-09-23T14:11:30Z","signature_id":"8c22befe458ceb42a7ccbeae384be919b61714799a47b6e0ea785ee3e8137ae4","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.0618556701030926,"anchor_occurrences":2,"after_last_push_count":63,"trend_counts":[97,19,4,0,0,1,2],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":81,"prior_last_seen_at":"2026-09-19T20:01:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-hv5jpf95qtjp/cluster-candidate-4-21-xnt7bq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f4a320\u003e: \n failed to create HCP cluster cluster-candidate-4-21-xnt7bq: failed waiting for cluster=\"cluster-candidate-4-21-xnt7bq\" in resourcegroup=\"rg-candidate-4-21-hv5jpf95qtjp\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\",\n \"name\": \"f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T20:49:22.499172304Z\",\n \"endTime\": \"2026-09-23T21:18:28.082375104Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m40s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-hv5jpf95qtjp/cluster-candidate-4-21-xnt7bq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f4a320\u003e: \n failed to create HCP cluster cluster-candidate-4-21-xnt7bq: failed waiting for cluster=\"cluster-candidate-4-21-xnt7bq\" in resourcegroup=\"rg-candidate-4-21-hv5jpf95qtjp\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\",\n \"name\": \"f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T20:49:22.499172304Z\",\n \"endTime\": \"2026-09-23T21:18:28.082375104Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m40s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001018240\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-57j9pfs2vmmr\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d3cf93e-ed99-43b6-aff0-026b8023aa5b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d3cf93e-ed99-43b6-aff0-026b8023aa5b\",\n \"name\": \"1d3cf93e-ed99-43b6-aff0-026b8023aa5b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T17:11:52.309512854Z\",\n \"endTime\": \"2026-09-23T17:40:54.426586313Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m20s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001018240\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-57j9pfs2vmmr\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d3cf93e-ed99-43b6-aff0-026b8023aa5b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d3cf93e-ed99-43b6-aff0-026b8023aa5b\",\n \"name\": \"1d3cf93e-ed99-43b6-aff0-026b8023aa5b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T17:11:52.309512854Z\",\n \"endTime\": \"2026-09-23T17:40:54.426586313Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m20s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","occurred_at":"2026-09-23T19:52:03Z","signature_id":"527306b3dde4b6e298931d98ad45937c8ba364a99fa4d7c4de2caf95207dcd18","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102795270910644224","occurred_at":"2026-09-23T16:20:35Z","signature_id":"2562b6a0aebd3dafa3041bae2bbef4d61a7d09096d6b568a3e7b98b9b1ca7da6","pr_number":7044,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.0618556701030926,"anchor_occurrences":2,"after_last_push_count":34,"trend_counts":[1,3,1,22,4,3,2],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":25,"prior_last_seen_at":"2026-09-20T23:42:35Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0018bec90\u003e: \n GET https://rp.aro-hcp-dev-shard3-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54444a02-0c9c-4321-935a-b1997c03724e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54444a02-0c9c-4321-935a-b1997c03724e\",\n \"name\": \"54444a02-0c9c-4321-935a-b1997c03724e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T15:55:02.945988001Z\",\n \"endTime\": \"2026-09-23T16:24:05.016015435Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0018bec90\u003e: \n GET https://rp.aro-hcp-dev-shard3-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54444a02-0c9c-4321-935a-b1997c03724e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54444a02-0c9c-4321-935a-b1997c03724e\",\n \"name\": \"54444a02-0c9c-4321-935a-b1997c03724e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T15:55:02.945988001Z\",\n \"endTime\": \"2026-09-23T16:24:05.016015435Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001078700\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-tsmxlnchqjdn\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a99324f-ced3-4b58-b157-e5f91f1fdc43\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a99324f-ced3-4b58-b157-e5f91f1fdc43\",\n \"name\": \"5a99324f-ced3-4b58-b157-e5f91f1fdc43\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T10:04:12.923013072Z\",\n \"endTime\": \"2026-09-23T10:33:15.61163171Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001078700\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-tsmxlnchqjdn\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a99324f-ced3-4b58-b157-e5f91f1fdc43\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a99324f-ced3-4b58-b157-e5f91f1fdc43\",\n \"name\": \"5a99324f-ced3-4b58-b157-e5f91f1fdc43\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T10:04:12.923013072Z\",\n \"endTime\": \"2026-09-23T10:33:15.61163171Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","occurred_at":"2026-09-23T14:54:53Z","signature_id":"e0cff06d2d3890e9b05a9d35c28b2aff9003a5f93ca966b808d2a0e175a745b6","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7077/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102689595517833216","occurred_at":"2026-09-23T09:20:17Z","signature_id":"7f41eb165d596d5dc7f2971efb22eb9df93680423e7b77c55b04839a62820eb7","pr_number":7077,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.0618556701030926,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001038980\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-bjdvsjn552jk\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\",\n \"name\": \"e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T00:25:27.786214979Z\",\n \"endTime\": \"2026-09-24T00:36:16.168471473Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-bjdvsjn552jk/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-16/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-bjdvsjn552jk/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20/subnets/customer-vnet-subnet-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-16/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001038980\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-bjdvsjn552jk\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\",\n \"name\": \"e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T00:25:27.786214979Z\",\n \"endTime\": \"2026-09-24T00:36:16.168471473Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-bjdvsjn552jk/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-16/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-bjdvsjn552jk/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20/subnets/customer-vnet-subnet-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-16/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7cd00\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-c9qhwvtrnfw9\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\",\n \"name\": \"cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T18:51:58.738258798Z\",\n \"endTime\": \"2026-09-23T19:01:09.161920644Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c9qhwvtrnfw9/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard2-03-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c9qhwvtrnfw9/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20/subnets/customer-vnet-subnet-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard2-03-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7cd00\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-c9qhwvtrnfw9\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\",\n \"name\": \"cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T18:51:58.738258798Z\",\n \"endTime\": \"2026-09-23T19:01:09.161920644Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c9qhwvtrnfw9/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard2-03-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c9qhwvtrnfw9/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20/subnets/customer-vnet-subnet-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard2-03-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102906434395049984","occurred_at":"2026-09-23T23:42:18Z","signature_id":"9d20053e21637cc884e3f5b506bcfcb59a5c2cd22b2f8ee5ff24cb97bfeb11c4","pr_number":7118,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102822640707178496","occurred_at":"2026-09-23T18:09:20Z","signature_id":"a7690e671288365fae42a54239a1fd3d4683bfffda4c651366b6b6d63b190ce6","pr_number":7118,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-7pwjpcnwnctq\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-7pwjpcnwnctq, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.0618556701030926,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-18T18:24:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:155]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0013ee480\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-bq2n97vrj5tz\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-bq2n97vrj5tz, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:155]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0013ee480\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-bq2n97vrj5tz\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-bq2n97vrj5tz, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001f44210\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-7pwjpcnwnctq\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-7pwjpcnwnctq, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001f44210\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-7pwjpcnwnctq\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-7pwjpcnwnctq, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","occurred_at":"2026-09-23T14:54:53Z","signature_id":"19b631f7d5f2a850bd1bdac6e41017fbcf045a75d8474e6adf13f1324cd43c53","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102570502110318592","occurred_at":"2026-09-23T01:27:26Z","signature_id":"75beab8f572182a387939937ea4f447a02998cc8ddb2ad7397436d6df460ffc0","pr_number":7090,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.0618556701030926,"anchor_occurrences":2,"after_last_push_count":44,"trend_counts":[31,15,4,4,2,2,2],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":90,"prior_last_seen_at":"2026-09-20T23:42:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":2}],"full_error_samples":["alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T21:01:37Z\n Ended: 2026-09-23T21:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9340032-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t0uqkogi5je8dvqpd30ug1p31mmvf4h-a9d6f3s9m6a1f1a\", pod=\"router-85d7d5999-qx49z\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t0uqkogi5je8dvqpd30ug1p31mmvf4h-a9d6f3s9m6a1f1a/router-85d7d5999-qx49z has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T06:11:33Z\n Ended: 2026-09-23T06:20:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3290112-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t0homk1jhbv7gpr72fmt0jahgksipbf-m8t7b7k5f6z0o9y\", pod=\"community-operators-catalog-5f9d56fbbf-bvthw\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t0homk1jhbv7gpr72fmt0jahgksipbf-m8t7b7k5f6z0o9y/community-operators-catalog-5f9d56fbbf-bvthw has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","occurred_at":"2026-09-23T19:52:03Z","signature_id":"9766aea3f661a15c4b1ef2c8ce948e7447e6a77d814fe12d510f6ddd5782ae00","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102624700403290112","occurred_at":"2026-09-23T05:02:00Z","signature_id":"fec44601492cd080815bfd1f2fda9d583e0a942b12d71ab0789e80aa75b0494f","pr_number":6922,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.0618556701030926,"anchor_occurrences":2,"after_last_push_count":15,"trend_counts":[1,4,1,4,4,3,2],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":24,"prior_last_seen_at":"2026-09-20T23:42:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":2}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T16:15:00Z\n Ended: 2026-09-23T16:46:01Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0621440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0621440/providers/microsoft.monitor/accounts/hcps-j0621440\", namespace=\"ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-jcrjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-23T16:15:00Z\n Ended: 2026-09-23T16:45:01Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0621440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0621440/providers/microsoft.monitor/accounts/hcps-j0621440\", namespace=\"ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-jcrjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T10:25:13Z\n Ended: 2026-09-23T10:52:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j7833216-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.79:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7833216/providers/microsoft.monitor/accounts/hcps-j7833216\", namespace=\"ocm-arohcpci01-2t0lc7i6ae878vg6j5amjnqk5ucv64ji-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-bj599\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t0lc7i6ae878vg6j5amjnqk5ucv64ji-cilium-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-23T10:25:14Z\n Ended: 2026-09-23T10:54:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j7833216-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.79:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7833216/providers/microsoft.monitor/accounts/hcps-j7833216\", namespace=\"ocm-arohcpci01-2t0lc7i6ae878vg6j5amjnqk5ucv64ji-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-bj599\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t0lc7i6ae878vg6j5amjnqk5ucv64ji-cilium-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","occurred_at":"2026-09-23T14:54:53Z","signature_id":"d57782e3eb6f6cd2059f92437161a764f82ac19056eed7d813da72181ad2f92d","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7077/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102689595517833216","occurred_at":"2026-09-23T09:20:17Z","signature_id":"acc0593d70344c6ce5881c51325fa15f3ec8bcecedf6ebedda37a38bcf852bf6","pr_number":7077,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.0618556701030926,"anchor_occurrences":2,"after_last_push_count":8,"trend_counts":[2,1,0,2,3,2,2],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":14,"prior_last_seen_at":"2026-09-20T23:42:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":2}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-23T23:23:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-23T23:23:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-23T23:31:17Z\n Ended: 2026-09-23T23:38:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"klusterlet-2t10muqh402iu8cpp36j8dibe4fqfcp3\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t10muqh402iu8cpp36j8dibe4fqfcp3/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-23T23:32:18Z\n Ended: 2026-09-23T23:38:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"klusterlet-2t10nsc5krmo0ki0tnfsecad8tsn2pgc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t10nsc5krmo0ki0tnfsecad8tsn2pgc/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","alert fired 13 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-23T16:16:22Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-23T16:16:22Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-23T16:27:22Z\n Ended: 2026-09-23T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg2he3nq1jn6qmjkr3aaspbgstbat\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t0qg2he3nq1jn6qmjkr3aaspbgstbat/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-23T16:27:22Z\n Ended: 2026-09-23T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg2h9j7mhi4jsf9pvdiin21umprdi\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t0qg2h9j7mhi4jsf9pvdiin21umprdi/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-23T16:27:22Z\n Ended: 2026-09-23T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg3radjabcqnahr40ucc01lfmk5a8\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t0qg3radjabcqnahr40ucc01lfmk5a8/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-23T16:27:22Z\n Ended: 2026-09-23T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qge4i1v30v9fissqi7v4cphc1oevm\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t0qge4i1v30v9fissqi7v4cphc1oevm/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-23T16:27:22Z\n Ended: 2026-09-23T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg2uevkmvfsffrb6l8hl332p8ardq\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t0qg2uevkmvfsffrb6l8hl332p8ardq/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-23T16:27:22Z\n Ended: 2026-09-23T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg504tgg42v8ako2cd4g5d2nos1ka\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t0qg504tgg42v8ako2cd4g5d2nos1ka/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-23T16:27:22Z\n Ended: 2026-09-23T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg55h7r4jqvn6q2ve0nvli480gqgj\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t0qg55h7r4jqvn6q2ve0nvli480gqgj/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-23T16:27:22Z\n Ended: 2026-09-23T16:33:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg4fmbf9m1ge8flue3mi2da0mjinv\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t0qg4fmbf9m1ge8flue3mi2da0mjinv/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-23T16:27:22Z\n Ended: 2026-09-23T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg4ehbc8a4kaevorrib9n0fhaggup\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t0qg4ehbc8a4kaevorrib9n0fhaggup/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-23T16:27:22Z\n Ended: 2026-09-23T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qge4qjfei7usbdki96o5frqdag2t4\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t0qge4qjfei7usbdki96o5frqdag2t4/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-23T16:27:22Z\n Ended: 2026-09-23T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg2pu5vkhroo89ii97sbdd2h2evli\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t0qg2pu5vkhroo89ii97sbdd2h2evli/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"7f9ed0d0bd04a4b470fe9147f2b70a32e6b959bfcc09d740b923b169f280b98f","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"e210a4373a8a1dff8335c12bcbfb22a4bca9ffd1f73200c066c63547ac34373a","pr_number":7108,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.0618556701030926,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[1,0,0,0,1,0,2],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-17T00:56:54Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":2}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-23T23:32:20Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-23T23:33:16Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-23T16:26:24Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-23T16:27:22Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"fbe2fd343d3a9197d314d9b85e39c50bd319b22385b4b05cd0ef79ae96f47f43","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"0b112a92c68becda58d472a3dbc17b517a7589cf579446aec092f675af664746","pr_number":7108,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.0618556701030926,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[1,0,0,1,1,3,2],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":13,"prior_last_seen_at":"2026-09-20T23:13:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-23T22:58:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-749558849-pbxrh\", region=\"westus3\", severity=\"warning\"\n Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-749558849-pbxrh has been in a non-ready state for longer than 5 minutes.","alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-23T01:52:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-ch8gg\", region=\"westus3\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-ch8gg has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-23T01:52:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-9ptjq\", region=\"westus3\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-9ptjq has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-23T01:52:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-kw5rf\", region=\"westus3\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-kw5rf has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-23T01:52:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-cxgg8\", region=\"westus3\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-cxgg8 has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"270a9593b205f1bd6398600a798ec9e0946dfbdebb1375f7feaf55d764c6e286","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","occurred_at":"2026-09-23T01:02:20Z","signature_id":"cd2acdca3a5c4271f034e124fe4ffb19ca10970d45a2e64d9db71f66d0c505d1","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b","failure_pattern":"alert [svc] userJourneyFrontendLatency6h30m fired","search_query":"alert [svc] userJourneyFrontendLatency6h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatency6h30m does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.0618556701030926,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,4,2],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatency6h30m does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-23T08:33:51Z\n Severity: Sev3\n Labels: alertname=\"userJourneyFrontendLatency6h30m\", cluster=\"ci01-j5147264-svc\", component=\"slo\", long_window=\"6h\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n Description: More than 6% of Frontend requests on cluster ci01-j5147264-svc exceeded 1s on both the 6h and 30m windows (6x burn of the 1% latency budget).","alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-23T08:39:18Z\n Severity: Sev3\n Labels: alertname=\"userJourneyFrontendLatency6h30m\", cluster=\"ci01-j3407488-svc\", component=\"slo\", long_window=\"6h\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n Description: More than 6% of Frontend requests on cluster ci01-j3407488-svc exceeded 1s on both the 6h and 30m windows (6x burn of the 1% latency budget)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651678955147264","occurred_at":"2026-09-23T06:50:00Z","signature_id":"01cbd1d336e53f82a367140655d108e641e536db1311bf518e94e91e812f8a75","pr_number":6687,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7099/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650881433407488","occurred_at":"2026-09-23T06:46:50Z","signature_id":"7b5a0fe8e7d62a0661702614b741d3d10ab9bf69cb3d6aad1f5d3a0807f9d130","pr_number":7099,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.0618556701030926,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[1,3,0,0,1,0,2],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":9,"prior_last_seen_at":"2026-09-18T08:07:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00121bf40\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:37971-\u003e172.30.0.10:53: read: connection refused\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00121bf40\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:37971-\u003e172.30.0.10:53: read: connection refused\n ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001a366b0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.20:59854-\u003e172.30.0.10:53: read: connection refused\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001a366b0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.20:59854-\u003e172.30.0.10:53: read: connection refused\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102818397388541952","occurred_at":"2026-09-23T17:52:29Z","signature_id":"b269b2a196f9ad9cd1702b095e9f309573b62770b7cab43403d5312cc9615be1","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7103/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102724367396900864","occurred_at":"2026-09-23T11:38:49Z","signature_id":"312be1f9783180426404eca8136660afa7c8940a066aacb369c351024c0827a4","pr_number":7103,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-v9rkf4k56z2c\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.0618556701030926,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[1,1,1,0,1,0,2],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":7,"prior_last_seen_at":"2026-09-19T10:22:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000a21860\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-v9rkf4k56z2c\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52906a0e-e465-4c26-861d-ab3698d37a5c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52906a0e-e465-4c26-861d-ab3698d37a5c\",\n \"name\": \"52906a0e-e465-4c26-861d-ab3698d37a5c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T18:28:44.991686145Z\",\n \"endTime\": \"2026-09-23T18:47:50.145290668Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000a21860\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-v9rkf4k56z2c\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52906a0e-e465-4c26-861d-ab3698d37a5c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52906a0e-e465-4c26-861d-ab3698d37a5c\",\n \"name\": \"52906a0e-e465-4c26-861d-ab3698d37a5c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T18:28:44.991686145Z\",\n \"endTime\": \"2026-09-23T18:47:50.145290668Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0010253a0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-fpvkvnrfcfj2\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1688403e-4bb7-4250-94fc-34a807ea0027\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1688403e-4bb7-4250-94fc-34a807ea0027\",\n \"name\": \"1688403e-4bb7-4250-94fc-34a807ea0027\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T13:47:50.262026457Z\",\n \"endTime\": \"2026-09-23T14:06:54.054886757Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0010253a0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-fpvkvnrfcfj2\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1688403e-4bb7-4250-94fc-34a807ea0027\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1688403e-4bb7-4250-94fc-34a807ea0027\",\n \"name\": \"1688403e-4bb7-4250-94fc-34a807ea0027\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T13:47:50.262026457Z\",\n \"endTime\": \"2026-09-23T14:06:54.054886757Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","occurred_at":"2026-09-23T17:26:00Z","signature_id":"48a8490fb0e062a5d2a53db5bf3f5a92fd4adef796973aafbfa795c3fe0dbe08","pr_number":7117,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","occurred_at":"2026-09-23T12:35:31Z","signature_id":"0f9515a2db56a29e5a383d5fa728e947a2b0cee316b599fe636e25ce8c008556","pr_number":6993,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"35d3eb882618c30032128a06359768c21f803eaffc0a5e443a0fbb171cbafbdb","failure_pattern":"Command failed after 5 attempts: oras cp quay.io/app-sre/aro-hcp-clusters-service@sha256:\u003chex\u003e arohcpsvcdev.azurecr.io/app-sre/aro-hcp-clusters-service:\u003chex\u003e --from-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json --to-registry-config","search_query":"Command failed after 5 attempts: oras cp quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/app-sre/aro-hcp-clusters-service:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.DLBEJUu7Ue/containers/auth.json --to-registry-config /tmp/tmp.DLBEJUu7Ue/containers/auth.json","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/global/mirror-image","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/global/mirror-image","occurrences":1}],"full_error_samples":["time=2026-09-23T14:16:23.495Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e clustersService.image.registry \u003cnil\u003e}:{\u003cnil\u003e clustersService.image.repository \u003cnil\u003e}@{\u003cnil\u003e clustersService.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-09-23T14:17:09.991Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 to arohcpsvcdev.azurecr.io/app-sre/aro-hcp-clusters-service:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6.\\nThe image will still be available under it's original digest sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 in the target registry.\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed after 5 attempts: oras cp quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/app-sre/aro-hcp-clusters-service:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.DLBEJUu7Ue/containers/auth.json --to-registry-config /tmp/tmp.DLBEJUu7Ue/containers/auth.json\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102753529973182464","occurred_at":"2026-09-23T13:34:43Z","signature_id":"15f55de3f2ea759d42d39cc08a545eae0c5c976acedd44ef9735b3b78e99695d","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":10,"prior_last_seen_at":"2026-09-20T19:59:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-ldnr6s\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00084af40\u003e: \n DNS for route host agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-ldnr6s\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00084af40\u003e: \n DNS for route host agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102853671329992704","occurred_at":"2026-09-23T20:12:38Z","signature_id":"5278202e490f8d8822dd976e22de925b821240a96d32e8bb0148f9031ab25a7c","pr_number":6969,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c8187cab875ebef3c2972bec23c80766d00e3b845998a754a4922f71837fefdf","failure_pattern":"ERROR CODE: DeploymentActive","search_query":"ERROR CODE: DeploymentActive","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:78]: failed to create customer resources for cilium CNI cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae83c0\u003e: \n failed to create customer-infra: failed creating deployment \"customer-infra-cilium-cl-kvs4jp\" in resourcegroup=\"cni-cilium-vbnz9zxk9mkz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/cni-cilium-vbnz9zxk9mkz/providers/Microsoft.Resources/deployments/customer-infra-cilium-cl-kvs4jp\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: DeploymentActive\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentActive\",\n \"message\": \"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-vbnz9zxk9mkz/providers/Microsoft.Resources/deployments/customer-infra-cilium-cl-kvs4jp' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '9/23/2026 8:41:53 PM' with correlationId '5bc6c0ba-196c-4242-b0a0-9410a2c0ee3b', and will expire at '9/30/2026 8:41:53 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:78]: failed to create customer resources for cilium CNI cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae83c0\u003e: \n failed to create customer-infra: failed creating deployment \"customer-infra-cilium-cl-kvs4jp\" in resourcegroup=\"cni-cilium-vbnz9zxk9mkz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/cni-cilium-vbnz9zxk9mkz/providers/Microsoft.Resources/deployments/customer-infra-cilium-cl-kvs4jp\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: DeploymentActive\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentActive\",\n \"message\": \"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-vbnz9zxk9mkz/providers/Microsoft.Resources/deployments/customer-infra-cilium-cl-kvs4jp' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '9/23/2026 8:41:53 PM' with correlationId '5bc6c0ba-196c-4242-b0a0-9410a2c0ee3b', and will expire at '9/30/2026 8:41:53 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","occurred_at":"2026-09-23T20:01:44Z","signature_id":"35009a2d8ec15af6629b3b447ac330d1c5376735efbdae49f03535446218b76a","pr_number":7110,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b76ddd45eba98c682618262a77ff4b0f94caf1aee78311c14f58f0e94282f506","failure_pattern":"ERROR CODE: InternalServerError; context failed to retrieve serial console logs for VM \"\u003cvm\u003e\"","search_query":"\"message\": \"Internal server error.\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:339]: failed to retrieve serial console logs for VM \"s0t7n3b8m8p4t6a-worker-pfp6w-rth7j\"\nUnexpected error:\n \u003c*errors.errorString | 0xc000a9b5e0\u003e: \n expected status 200 OK, got 500: {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:339]: failed to retrieve serial console logs for VM \"s0t7n3b8m8p4t6a-worker-pfp6w-rth7j\"\nUnexpected error:\n \u003c*errors.errorString | 0xc000a9b5e0\u003e: \n expected status 200 OK, got 500: {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102790414846136320","occurred_at":"2026-09-23T16:00:38Z","signature_id":"2074bff381a20809ab3d0b56df62ec1c9c4cecad07bc3b02a8f9d837a90a1b6a","pr_number":7112,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[3,2,0,0,0,0,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":10,"prior_last_seen_at":"2026-09-18T03:01:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010240a0\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-w2f9x55rndpg\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/269f9f7f-d982-42df-9b95-a4f0fcdc404f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/269f9f7f-d982-42df-9b95-a4f0fcdc404f\",\n \"name\": \"269f9f7f-d982-42df-9b95-a4f0fcdc404f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T14:21:22.155873772Z\",\n \"endTime\": \"2026-09-23T14:22:05.314078185Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010240a0\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-w2f9x55rndpg\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/269f9f7f-d982-42df-9b95-a4f0fcdc404f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/269f9f7f-d982-42df-9b95-a4f0fcdc404f\",\n \"name\": \"269f9f7f-d982-42df-9b95-a4f0fcdc404f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T14:21:22.155873772Z\",\n \"endTime\": \"2026-09-23T14:22:05.314078185Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102750533910532096","occurred_at":"2026-09-23T13:22:24Z","signature_id":"bccb01e525eebedfd6d8ea72bf2f185117e64e499bd98913c911974096d95997","pr_number":6439,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b6fc32628c7943fbfb2873910bf615def7fec7cbe037b84d079cb918fd32cbe8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, cluster-network-operator, dns-operator, ingress-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-ks79lqvdvdz4/cluster-candidate-4-21-tjs922 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b96700\u003e: \n failed to create HCP cluster cluster-candidate-4-21-tjs922: failed waiting for cluster=\"cluster-candidate-4-21-tjs922\" in resourcegroup=\"rg-candidate-4-21-ks79lqvdvdz4\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f164752-446b-48ce-a79b-2e5f890f02c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f164752-446b-48ce-a79b-2e5f890f02c6\",\n \"name\": \"6f164752-446b-48ce-a79b-2e5f890f02c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T20:08:29.367613866Z\",\n \"endTime\": \"2026-09-23T20:37:38.881566969Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, packageserver, olm-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-ks79lqvdvdz4/cluster-candidate-4-21-tjs922 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b96700\u003e: \n failed to create HCP cluster cluster-candidate-4-21-tjs922: failed waiting for cluster=\"cluster-candidate-4-21-tjs922\" in resourcegroup=\"rg-candidate-4-21-ks79lqvdvdz4\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f164752-446b-48ce-a79b-2e5f890f02c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f164752-446b-48ce-a79b-2e5f890f02c6\",\n \"name\": \"6f164752-446b-48ce-a79b-2e5f890f02c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T20:08:29.367613866Z\",\n \"endTime\": \"2026-09-23T20:37:38.881566969Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, packageserver, olm-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840286248964096","occurred_at":"2026-09-23T19:19:27Z","signature_id":"3fec0289f4e859b2672b4ecfd1c53de11f801ca25e568cb6fd49601565fabc04","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"750410b055b83de5666e72039e788f64cc42e64b3191a432c4cf3efab4dc693f","failure_pattern":"Operation cannot be fulfilled on nodes \"\u003cnode\u003e\": the object has been modified; please apply your changes to the latest version and try again","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-20T09:53:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node t9c6r3e7k2o1w9o-cordon-np-st9vc-2hvv8\nUnexpected error:\n \u003c*errors.StatusError | 0xc000ca99a0\u003e: \n Operation cannot be fulfilled on nodes \"t9c6r3e7k2o1w9o-cordon-np-st9vc-2hvv8\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node t9c6r3e7k2o1w9o-cordon-np-st9vc-2hvv8\nUnexpected error:\n \u003c*errors.StatusError | 0xc000ca99a0\u003e: \n Operation cannot be fulfilled on nodes \"t9c6r3e7k2o1w9o-cordon-np-st9vc-2hvv8\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","occurred_at":"2026-09-23T17:26:00Z","signature_id":"23493215ee1a820f0efaa9de36d3f924d4bf22faf75dcf25d674d9fd4bee714f","pr_number":7117,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":9,"trend_counts":[1,3,1,1,2,2,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":13,"prior_last_seen_at":"2026-09-20T23:13:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-23T16:17:56Z\n Ended: 2026-09-23T16:44:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0621440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0621440/providers/microsoft.monitor/accounts/hcps-j0621440\", namespace=\"ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"b0ac0532-43d4-43f8-be90-7de951bc2ef5\"\n Description: Pod ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-09-23T16:17:56Z\n Ended: 2026-09-23T16:45:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0621440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0621440/providers/microsoft.monitor/accounts/hcps-j0621440\", namespace=\"ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"b0ac0532-43d4-43f8-be90-7de951bc2ef5\"\n Description: Pod ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","occurred_at":"2026-09-23T14:54:53Z","signature_id":"d679a18a9e7a13bb82f0cbe4fdbabac5624367c05fd9b68e2584227db8962e2f","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,3,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-16T21:13:51Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-23T02:02:02Z\n Severity: Sev3\n Labels: alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j0769280-svc\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"warning\"\n Description: The Arobit forwarder metrics endpoint on cluster ci01-j0769280-svc has been unreachable for 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","occurred_at":"2026-09-23T01:02:20Z","signature_id":"0bff05a4b68cd34010dac9320f26f4a4adb189546a09d7a89930f18e9ffb6349","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,3,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":9,"prior_last_seen_at":"2026-09-16T21:13:51Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-23T02:02:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.47:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0769280/providers/microsoft.monitor/accounts/services-j0769280\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-wqtm5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-23T02:02:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.47:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0769280/providers/microsoft.monitor/accounts/services-j0769280\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-wqtm5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","occurred_at":"2026-09-23T01:02:20Z","signature_id":"80bbd3a02e54b8fc7971960324a0123e49b270a5d705be23d017d8e1a4b2f32e","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[0,2,0,2,0,4,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":12,"prior_last_seen_at":"2026-09-20T12:02:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00190deb0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00190deb0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102700766367059968","occurred_at":"2026-09-23T10:05:03Z","signature_id":"80e228d9d0c46c7582a7eb683b187f1c62b13a5fdc9e873f52821f9b7629b50e","pr_number":7065,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1ffc144947159d20ed7dd5010cb0a93b810641281a5d037d76b1920de5d6d537","failure_pattern":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later","search_query":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-18T00:42:37Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001cc06c0\u003e: \n clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001cc06c0\u003e: \n clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102902545511878656","occurred_at":"2026-09-23T23:26:51Z","signature_id":"c8ce66f37f9d13dc657060400db223a90301ac4438226c58a0c01cac44d1ec13","pr_number":5562,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"61e8157ef3845c610ccadbb9e49365372ea07610a9b75316502d11b44c89c896","failure_pattern":"expected all guest nodes to be unschedulable, schedulable=\u003cnode\u003e, \u003cnode\u003e; nodes: \u003cnode\u003e ready=true unschedulable=true created=\u003ctimestamp\u003e nodePool=\u003cnodepool\u003e; \u003cnode\u003e ready=true unschedulable=false created=\u003ctimestamp\u003e nodePool=\u003cnodepool\u003e; \u003cnode\u003e ready=false","search_query":"expected all guest nodes to be unschedulable, schedulable=e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv, e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r; nodes: e6r1q1p6z7w9f4r-np-dataplane-fqlcw-4wrz9 ready=true unschedulable=true created=2026-09-23T07:52:57Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv ready=true unschedulable=false created=2026-09-23T08:05:13Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r ready=false unschedulable=false created=2026-09-23T08:11:10Z nodePool=e6r1q1p6z7w9f4r-np-dataplane","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when dataplane is degraded from 4.20 minor to 4.21 minor with cordoned nodes","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when dataplane is degraded from 4.20 minor to 4.21 minor with cordoned nodes","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_dataplane.go:304]: guest nodes were no longer unschedulable after upgrade on cluster \"cp-ystream-dp-cordon-4-21-8ndsmf\"\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0013005c0\u003e: \n expected all guest nodes to be unschedulable, schedulable=e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv, e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r; nodes: e6r1q1p6z7w9f4r-np-dataplane-fqlcw-4wrz9 ready=true unschedulable=true created=2026-09-23T07:52:57Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv ready=true unschedulable=false created=2026-09-23T08:05:13Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r ready=false unschedulable=false created=2026-09-23T08:11:10Z nodePool=e6r1q1p6z7w9f4r-np-dataplane\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_dataplane.go:304]: guest nodes were no longer unschedulable after upgrade on cluster \"cp-ystream-dp-cordon-4-21-8ndsmf\"\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0013005c0\u003e: \n expected all guest nodes to be unschedulable, schedulable=e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv, e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r; nodes: e6r1q1p6z7w9f4r-np-dataplane-fqlcw-4wrz9 ready=true unschedulable=true created=2026-09-23T07:52:57Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv ready=true unschedulable=false created=2026-09-23T08:05:13Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r ready=false unschedulable=false created=2026-09-23T08:11:10Z nodePool=e6r1q1p6z7w9f4r-np-dataplane\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651461333684224","occurred_at":"2026-09-23T06:49:08Z","signature_id":"a02ab509af0dce2add587b938aa804dece2558ca7621ffc208e009f99dae4089","pr_number":6970,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"acb0571a19c874c4edfc6efce4cf429757c85c1334f4e11589a3958d721e5d38","failure_pattern":"failed to roll out Helm release: \"fleet-reg-ci01-j6376960-mgmt-1\" has no deployed releases","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: \\\"fleet-reg-ci01-j6376960-mgmt-1\\\" has no deployed releases","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/svc-stamped/deploy (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/svc-stamped/deploy (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-09-23T18:34:52.490Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-23T18:34:52.977Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 values=\"map[cloudEnvironmentName:AzurePublicCloud cosmos:map[name:arohcpci01-rp-j6376960 url:https://arohcpci01-rp-j6376960.documents.azure.com:443/] image:map[digest:sha256:6bc5ed31e42afccc2abd6b21b817cc402be829c20f8cc00e71fde01c9be1523a registry:arohcpsvcdev.azurecr.io repository:ci-op-6hlkxni0/pipeline] registration:map[aksResourceID:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6376960-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j6376960-mgmt-1 autoApprove:true hostedClustersManagedIdentitiesKeyVaultURL:https://ah-ci01-mi-j6376960-1.vault.azure.net/ hostedClustersSecretsKeyVaultManagedIdentityClientID:b2cc94cd-3e8b-42a9-bf2a-e025837a95cd hostedClustersSecretsKeyVaultURL:https://ah-ci01-cx-j6376960-1.vault.azure.net/ kubeApplierCosmosContainerName:Manifests-MC-1 maestroConsumerName:hcp-underlay-aro-hcp-dev-shard2-slot-01-mgmt-1 maestroGRPCTarget:maestro-grpc.maestro.svc.cluster.local:8090 maestroRESTAPIURL:http://maestro.maestro.svc.cluster.local:8000 provisionShardStatus:active publicDNSZoneResourceID:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6376960/providers/Microsoft.Network/dnsZones/aro-hcp-dev-shard2-slot-01.hcp.osadev.cloud stampIdentifier:1] serviceAccountName:fleet]\"\ntime=2026-09-23T18:34:52.977Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1\ntime=2026-09-23T18:34:54.319Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 namespace=fleet\ntime=2026-09-23T18:34:54.894Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 dryRun=false\ntime=2026-09-23T18:34:54.894Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1\ntime=2026-09-23T18:35:05.401Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 numVersions=0\ntime=2026-09-23T18:35:05.530Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 err=\"\\\"fleet-reg-ci01-j6376960-mgmt-1\\\" has no deployed releases\"\ntime=2026-09-23T18:35:05.530Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: \\\"fleet-reg-ci01-j6376960-mgmt-1\\\" has no deployed releases\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102817746126376960","occurred_at":"2026-09-23T17:49:53Z","signature_id":"aabbcc585420cff60362100bdaf09336ff0307d5c296c6855ec0f5fe21f850ce","pr_number":6915,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,3,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":6,"prior_last_seen_at":"2026-09-16T09:56:19Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","occurrences":1}],"full_error_samples":["time=2026-09-23T01:45:47.799Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-23T01:45:48.393Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0769280-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j0769280-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:1190e8af-ea6a-4410-bc47-5facec337dc4 region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#1190e8af-ea6a-4410-bc47-5facec337dc4 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-23T01:45:48.393Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:45:48.667Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-23T01:45:49.209Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-23T01:45:49.209Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:45:49.334Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:55:54.330Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T01:55:54.331Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T01:56:54.333Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-23T01:56:55.421Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0769280-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j0769280-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:1190e8af-ea6a-4410-bc47-5facec337dc4 region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#1190e8af-ea6a-4410-bc47-5facec337dc4 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-23T01:56:55.421Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:56:55.777Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-23T01:56:56.324Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:56:56.324Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:56:56.630Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-23T01:56:57.690Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:56:57.691Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:56:58.299Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:56:58.862Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T01:56:59.423Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T01:57:00.012Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:57:00.364Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:57:00.910Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:57:01.301Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-23T01:57:01.301Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:57:01.546Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-23T02:07:04.175Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T02:07:04.175Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T02:08:04.175Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-23T02:08:05.008Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0769280-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j0769280-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:1190e8af-ea6a-4410-bc47-5facec337dc4 region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#1190e8af-ea6a-4410-bc47-5facec337dc4 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-23T02:08:05.008Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:05.466Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-23T02:08:06.204Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:06.204Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:06.382Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-23T02:08:07.458Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:07.458Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:07.843Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:08.593Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T02:08:09.447Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T02:08:09.853Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:10.521Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:11.194Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:11.889Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-23T02:08:11.889Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:12.134Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-23T02:15:47.989Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-23T02:15:47.990Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","occurred_at":"2026-09-23T01:02:20Z","signature_id":"f020f9342f43cace837cc83b8952affa433a8c7410f5746e59533d769ed278b9","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d91796eeda5752e6e2b49b398ec435f90a4ab6d3394ea2d0cd4ce5b2c2f30a11","failure_pattern":"not all KSM metrics appeared in Azure Monitor","search_query":"not all KSM metrics appeared in Azure Monitor","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics metrics should be present in HCP Azure Monitoring Workspace","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics metrics should be present in HCP Azure Monitoring Workspace","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:147]: Timed out after 900.006s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:143 with:\nexpected ingresscontroller_info metrics from kube-state-metrics but got no results\nExpected\n \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:147]: Timed out after 900.006s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:143 with:\nexpected ingresscontroller_info metrics from kube-state-metrics but got no results\nExpected\n \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102774033853452288","occurred_at":"2026-09-23T14:56:11Z","signature_id":"3812816f7f5de01152a0f4bc5ce516ed73308e2f6080e009a2d35e56b303c38c","pr_number":7009,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4e0bd8e13cb37d9a0194e77a0943faa52db03c4ceb721253ee7a21fbde2f97e0","failure_pattern":"run command \"e2e-runcommand-1790197778134945960\" failed with exit code 6 on VM \"\u003cvm\u003e\" (output: \"000\")","search_query":"run command \"e2e-runcommand-1790197778134945960\" failed with exit code 6 on VM \"private-ingress-test-vm\" (output: \"000\")","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 600.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n \u003c*errors.errorString | 0xc002901d30\u003e: \n run command \"e2e-runcommand-1790197778134945960\" failed with exit code 6 on VM \"private-ingress-test-vm\" (output: \"000\")\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 600.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n \u003c*errors.errorString | 0xc002901d30\u003e: \n run command \"e2e-runcommand-1790197778134945960\" failed with exit code 6 on VM \"private-ingress-test-vm\" (output: \"000\")\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","occurred_at":"2026-09-23T20:01:44Z","signature_id":"05cb3f86a6f99432ca7873642f1755e0d1de74704c579e10519fe31b0a3ab239","pr_number":7110,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8eaae2745978f98ae080df0fcc5b37bd11dd539de8c53dcf70259bcbe794a620","failure_pattern":"{\"level\":\"ERROR\",\"msg\":\"run failed\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j5982976-mgmt-2\",\"clusterName\":\"ci01-j5982976-mgmt-2\",\"region\":\"westus3\",\"err\":\"waiting for deployment","search_query":"{\\\"time\\\":\\\"2026-09-23T20:10:16.314392598Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: Get \\\\\\\"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5982976-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584114129632610784?api-version=2025-04-01\u0026t=639257907264738192\u0026c=MIIIJzCCBw-gAwIBAgIRAKCxVbScFDiCR8oT6qcGYJMwDQYJKoZIhvcNAQELBQAwNjE0MDIGA1UEAxMrQ0NNRSBHMSBUTFMgUlNBIDIwNDggU0hBMjU2IDIwNDkgV0NVUyBDQSAwMTAeFw0yNjA5MTkwMDU4MzJaFw0yNjEyMTQwNjU4MzJaMEAxPjA8BgNVBAMTNWFzeW5jb3BlcmF0aW9uc2lnbmluZ2NlcnRpZmljYXRlLm1hbmFnZW1lbnQuYXp1cmUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzFNlTnLezYuVV9JBgdhqA91jRJUtR4AjUkUKP5Nk7STlDt0_Hwx2OHYvMcuSX2B7klrS1nxF26zm4lnQKcCJTslJYfDrx756nLb5iGS6OS0koE0rze5-sUiu_IHHt28tpGv_9lvJvRWPDZc1uYCzrGyBpz6HvBzVgl3BeyWo9RTZYJhgBVNIATJuDRUzovVt8XczDob4euvsqOVr7rBx6wvKf79ASaeXB6PbT19HGywTrd9Xkd33sWfzQu5ZuHtSgX3i1sp_4BVE8VTuCkqIku5kzbulG3jvlSrsOmn680vsH3sS1APY1L7SymeuyNuVIv20HZmQukmjreOzZu-iGQIDAQABo4IFJDCCBSAwgZ0GA1UdIASBlTCBkjAMBgorBgEEAYI3ewEBMGYGCisGAQQBgjd7AgIwWDBWBggrBgEFBQcCAjBKHkgAMwAzAGUAMAAxADkAMgAxAC0ANABkADYANAAtADQAZgA4AGMALQBhADAANQA1AC0ANQBiAGQAYQBmAGYAZAA1AGUAMwAzAGQwDAYKKwYBBAGCN3sDAjAMBgorBgEEAYI3ewQCMAwGA1UdEwEB_wQCMAAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA4GA1UdDwEB_wQEAwIFoDAdBgNVHQ4EFgQU7SRzDdTHXndOcavyYsWupdx4MecwHwYDVR0jBBgwFoAUFNI34PbWfX7djbq6ZasElCXglh0wggH7BgNVHR8EggHyMIIB7jB7oHmgd4Z1aHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3RjZW50cmFsdXMvY3Jscy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMH2ge6B5hndodHRwOi8vc2Vjb25kYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0Y2VudHJhbHVzL2NybHMvY2NtZXdlc3RjZW50cmFsdXNwa2kvY2NtZXdlc3RjZW50cmFsdXNpY2EwMS80MS9jdXJyZW50LmNybDBsoGqgaIZmaHR0cDovL2NybC5taWNyb3NvZnQuY29tL3dlc3RjZW50cmFsdXMvY3Jscy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMIGBoH-gfYZ7aHR0cDovL2NjbWV3ZXN0Y2VudHJhbHVzcGtpLndlc3RjZW50cmFsdXMucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMIICAAYIKwYBBQUHAQEEggHyMIIB7jB-BggrBgEFBQcwAoZyaHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3RjZW50cmFsdXMvY2FjZXJ0cy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxL2NlcnQuY2VyMIGABggrBgEFBQcwAoZ0aHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdGNlbnRyYWx1cy9jYWNlcnRzL2NjbWV3ZXN0Y2VudHJhbHVzcGtpL2NjbWV3ZXN0Y2VudHJhbHVzaWNhMDEvY2VydC5jZXIwbwYIKwYBBQUHMAKGY2h0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0Y2VudHJhbHVzL2NhY2VydHMvY2NtZXdlc3RjZW50cmFsdXNwa2kvY2NtZXdlc3RjZW50cmFsdXNpY2EwMS9jZXJ0LmNlcjB4BggrBgEFBQcwAoZsaHR0cDovL2NjbWV3ZXN0Y2VudHJhbHVzcGtpLndlc3RjZW50cmFsdXMucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdGNlbnRyYWx1c2ljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQA5ody_Abz9ik5_3WVrjOb7e5cyPkeuDRHrpQ3VO0TeurG5LwTtQ-Yu-lE3ow2Qq4IbIIP6kpzWMQ_woGCp96pGdwvp7vLu4SYc-Rp-CKqLpJtXBRQ01RDbqBZWPwrrgrfMjTaopq1rEEHOBBqt2X4fPXlf91DULpbYJikwKVt4rSQPFgvzjtIpRSeHkIIw8oA95SBfgvs-CIisoc1h-1J1s6jmYw8Kqgqc-NoVLrvf4qRHrd8_0YK2nmtl5XnHD8a50gNVozHfx52l7NfwV_tYfwIO82lHrNxrpLkb7gSAAoAkma6s2Tjh23fpSj-E9OFOFMeuCwnaykbAggIZp9hc\u0026s=xAFEUIMkN7HqEcaWnjPt98SPv2n8tDGNfsg9g3ETUj427eutsI2z-oSn-NYDEALQ9cMRKqky-VWvsQV0qp29lgMWk_hw47hsrPBP4zu666M_n4lLgU8QDrRKLDzrIYelxUd035HLMh9F5fWNHrGNulgbDCkeZeqOoc6EAne-535SWhie6nA952GikZdg_fD2pGBdL53D40Ycq2WqzPFAtYN4D-nKCFWKG8QoWd59PYeAW-C_ZWVMdATP9LmSEVVMHqMF13HK7-5WRYDFxs-LPKfY7okL22TjrA2RLKxuv1xWiSUxb0QLisSFa7Bqvbbl6syA1C3_0qKLeHs_XDklIA\u0026h=dLBpH5niUgnkOleLXSR6xT4LrPMGJuQLdmK3gutCEoY\\\\\\\": stream error: stream ID 33; CANCEL; received from peer\\\",\\\"elapsed\\\":\\\"4m56.832387486s\\\"}","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.0309278350515463,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-23T20:05:19.475Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-23T20:10:16.316Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-23T20:05:19.482013132Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-09-23T20:05:21.259341341Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-23T20:05:21.679397764Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-23T20:10:16.314281955Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"collecting deployment operations\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"err\\\":\\\"Get \\\\\\\"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5982976-mgmt-2/deployments/aks-cluster-create/operations?api-version=2025-04-01\\\\\\\": stream error: stream ID 41; CANCEL; received from peer\\\"}\\n{\\\"time\\\":\\\"2026-09-23T20:10:16.314392598Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: Get \\\\\\\"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5982976-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584114129632610784?api-version=2025-04-01\u0026t=639257907264738192\u0026c=MIIIJzCCBw-gAwIBAgIRAKCxVbScFDiCR8oT6qcGYJMwDQYJKoZIhvcNAQELBQAwNjE0MDIGA1UEAxMrQ0NNRSBHMSBUTFMgUlNBIDIwNDggU0hBMjU2IDIwNDkgV0NVUyBDQSAwMTAeFw0yNjA5MTkwMDU4MzJaFw0yNjEyMTQwNjU4MzJaMEAxPjA8BgNVBAMTNWFzeW5jb3BlcmF0aW9uc2lnbmluZ2NlcnRpZmljYXRlLm1hbmFnZW1lbnQuYXp1cmUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzFNlTnLezYuVV9JBgdhqA91jRJUtR4AjUkUKP5Nk7STlDt0_Hwx2OHYvMcuSX2B7klrS1nxF26zm4lnQKcCJTslJYfDrx756nLb5iGS6OS0koE0rze5-sUiu_IHHt28tpGv_9lvJvRWPDZc1uYCzrGyBpz6HvBzVgl3BeyWo9RTZYJhgBVNIATJuDRUzovVt8XczDob4euvsqOVr7rBx6wvKf79ASaeXB6PbT19HGywTrd9Xkd33sWfzQu5ZuHtSgX3i1sp_4BVE8VTuCkqIku5kzbulG3jvlSrsOmn680vsH3sS1APY1L7SymeuyNuVIv20HZmQukmjreOzZu-iGQIDAQABo4IFJDCCBSAwgZ0GA1UdIASBlTCBkjAMBgorBgEEAYI3ewEBMGYGCisGAQQBgjd7AgIwWDBWBggrBgEFBQcCAjBKHkgAMwAzAGUAMAAxADkAMgAxAC0ANABkADYANAAtADQAZgA4AGMALQBhADAANQA1AC0ANQBiAGQAYQBmAGYAZAA1AGUAMwAzAGQwDAYKKwYBBAGCN3sDAjAMBgorBgEEAYI3ewQCMAwGA1UdEwEB_wQCMAAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA4GA1UdDwEB_wQEAwIFoDAdBgNVHQ4EFgQU7SRzDdTHXndOcavyYsWupdx4MecwHwYDVR0jBBgwFoAUFNI34PbWfX7djbq6ZasElCXglh0wggH7BgNVHR8EggHyMIIB7jB7oHmgd4Z1aHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3RjZW50cmFsdXMvY3Jscy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMH2ge6B5hndodHRwOi8vc2Vjb25kYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0Y2VudHJhbHVzL2NybHMvY2NtZXdlc3RjZW50cmFsdXNwa2kvY2NtZXdlc3RjZW50cmFsdXNpY2EwMS80MS9jdXJyZW50LmNybDBsoGqgaIZmaHR0cDovL2NybC5taWNyb3NvZnQuY29tL3dlc3RjZW50cmFsdXMvY3Jscy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMIGBoH-gfYZ7aHR0cDovL2NjbWV3ZXN0Y2VudHJhbHVzcGtpLndlc3RjZW50cmFsdXMucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMIICAAYIKwYBBQUHAQEEggHyMIIB7jB-BggrBgEFBQcwAoZyaHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3RjZW50cmFsdXMvY2FjZXJ0cy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxL2NlcnQuY2VyMIGABggrBgEFBQcwAoZ0aHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdGNlbnRyYWx1cy9jYWNlcnRzL2NjbWV3ZXN0Y2VudHJhbHVzcGtpL2NjbWV3ZXN0Y2VudHJhbHVzaWNhMDEvY2VydC5jZXIwbwYIKwYBBQUHMAKGY2h0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0Y2VudHJhbHVzL2NhY2VydHMvY2NtZXdlc3RjZW50cmFsdXNwa2kvY2NtZXdlc3RjZW50cmFsdXNpY2EwMS9jZXJ0LmNlcjB4BggrBgEFBQcwAoZsaHR0cDovL2NjbWV3ZXN0Y2VudHJhbHVzcGtpLndlc3RjZW50cmFsdXMucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdGNlbnRyYWx1c2ljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQA5ody_Abz9ik5_3WVrjOb7e5cyPkeuDRHrpQ3VO0TeurG5LwTtQ-Yu-lE3ow2Qq4IbIIP6kpzWMQ_woGCp96pGdwvp7vLu4SYc-Rp-CKqLpJtXBRQ01RDbqBZWPwrrgrfMjTaopq1rEEHOBBqt2X4fPXlf91DULpbYJikwKVt4rSQPFgvzjtIpRSeHkIIw8oA95SBfgvs-CIisoc1h-1J1s6jmYw8Kqgqc-NoVLrvf4qRHrd8_0YK2nmtl5XnHD8a50gNVozHfx52l7NfwV_tYfwIO82lHrNxrpLkb7gSAAoAkma6s2Tjh23fpSj-E9OFOFMeuCwnaykbAggIZp9hc\u0026s=xAFEUIMkN7HqEcaWnjPt98SPv2n8tDGNfsg9g3ETUj427eutsI2z-oSn-NYDEALQ9cMRKqky-VWvsQV0qp29lgMWk_hw47hsrPBP4zu666M_n4lLgU8QDrRKLDzrIYelxUd035HLMh9F5fWNHrGNulgbDCkeZeqOoc6EAne-535SWhie6nA952GikZdg_fD2pGBdL53D40Ycq2WqzPFAtYN4D-nKCFWKG8QoWd59PYeAW-C_ZWVMdATP9LmSEVVMHqMF13HK7-5WRYDFxs-LPKfY7okL22TjrA2RLKxuv1xWiSUxb0QLisSFa7Bqvbbl6syA1C3_0qKLeHs_XDklIA\u0026h=dLBpH5niUgnkOleLXSR6xT4LrPMGJuQLdmK3gutCEoY\\\\\\\": stream error: stream ID 33; CANCEL; received from peer\\\",\\\"elapsed\\\":\\\"4m56.832387486s\\\"}\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102846182265982976","occurred_at":"2026-09-23T19:42:53Z","signature_id":"b83201cdb7e435a0abab8639b944505f58098f612295a321d14698e824fe6511","pr_number":6925,"after_last_push_of_merged_pr":false}]}]},{"environment":"int","summary":{"total_runs":3,"failed_runs":2,"raw_occurrences":2,"matched_occurrences":2,"runs_affected":2},"rows":[{"environment":"int","failure_pattern_id":"09d6899fcee106147461837dfde3776fb25ff9f2af5b0d715a24d5e92bbbf532","failure_pattern":"cluster CA data must contain exactly one PEM block","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":66.66666666666667,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block\nExpected\n \u003c[]uint8 | len:6010, cap:6010\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block\nExpected\n \u003c[]uint8 | len:6010, cap:6010\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102710804108283904","occurred_at":"2026-09-23T10:44:56Z","signature_id":"0513c9dba1d81c84b15ffbc8b39cea7d4f8ffd6ade45b91dfec359ec896f33e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102650382160236544","occurred_at":"2026-09-23T06:44:51Z","signature_id":"0513c9dba1d81c84b15ffbc8b39cea7d4f8ffd6ade45b91dfec359ec896f33e5","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"raw_occurrences":28,"matched_occurrences":28,"runs_affected":2},"rows":[{"environment":"prod","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":14,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":14,"after_last_push_count":14,"trend_counts":[0,0,0,0,0,0,14],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f96700\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-nfm8pxmkddmn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-nfm8pxmkddmn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f96700\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-nfm8pxmkddmn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-nfm8pxmkddmn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001010900\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-2lsdzccng668/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001010900\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-2lsdzccng668/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000618ec0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5n2hfst68hj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-b5n2hfst68hj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000618ec0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5n2hfst68hj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-b5n2hfst68hj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"696c58cbd3aa8d300062df4a47583e3f809311c672a43e3da76b7164514ff28c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"80af8612530648ca50b987b6f9cdbcfec5e5733a4eb73b613f4f2cca40440dda","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"8b972b568eeb592cb7c4e3ea2dd32eb95af26dd5cf16683b957140e9cded71c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"9099db8102021ee82d4f7fa885ff9d6c40c7a777e05056ac01595d8cbe6b71e0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"a15265dd30984b38cf2e97f6a7e9fa79ca18ae3cedd2d533ed100ef489695a0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"a6b4d2a29f145149ab937c5195b130875034ebc08e0235a73002c81ef5d5096f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"e5a3278c01c0abce683e00630bbc784dbd610e4e57d5f23368cb17f09c05ad78","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"3c47413f0338789d70091286ea56544fcdee01d67131f0f7851a0b97474d7105","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"60163a0f600eba5c06ff2c878eb2a0092d005dbebf2c0e82fe91feb85ffe2ec7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"950be739fb33f7b6adecbfc5254b9a9b8d54463fd0cd14ad47b8f998fd5afdc9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"a1b9cf6305c1912800fbad4a1b18908456f24a8e3cd6243102d4426c2fb0976f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"b2a0aa0f0d5efc781c39edb1e340969ee91b48c4052331e8dd71b8357572e91b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"ef884a233cc88172af6105a108535d3de27389d5003c528f736c27bc0329a063","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"f40ba7ba94d5d5cc9b8228f74ef22fcc6c8cf1de075f86ae5b726e2a4ba76bcb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":4,"after_last_push_count":38,"trend_counts":[4,5,5,5,5,5,4],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-20T23:00:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002a7cd50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-lqwwzcqjkl4d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002a7cd50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-lqwwzcqjkl4d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002564810\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-cr6zxd6vcc7j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002564810\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-cr6zxd6vcc7j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001570180\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-9vczdt9qztzh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001570180\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-9vczdt9qztzh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"0b19163b0950f2bc9dac2fa7bbe2fe9b7e8b4eddb5161aef3f7f5ec4427662e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"3532da433aa865198e7af69b2d67f9f685db4cc1befc31928db022dff5c8ba6f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"4beec4b23a49b2c4afc633ee4de74ca68b43b8c0a169e74f6e8c71ff427a668d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"c0473fca37809ede533480e60205fed102ad16eccbd4d3d6441046e31b66bf9d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":4,"after_last_push_count":28,"trend_counts":[4,4,4,4,4,4,4],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":7,"prior_last_seen_at":"2026-09-20T23:00:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cca500\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-4fchkp\" in resourcegroup=\"rg-np-skip-minor-4fchkp-d9wq5jxbr4vg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-4fchkp-d9wq5jxbr4vg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-4fchkp/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cca500\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-4fchkp\" in resourcegroup=\"rg-np-skip-minor-4fchkp-d9wq5jxbr4vg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-4fchkp-d9wq5jxbr4vg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-4fchkp/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00121ed00\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-zj2nbd\" resourcegroup=\"rg-np-dg-minor-zj2nbd-nbv87cpm7mgj\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-zj2nbd-nbv87cpm7mgj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-zj2nbd/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00121ed00\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-zj2nbd\" resourcegroup=\"rg-np-dg-minor-zj2nbd-nbv87cpm7mgj\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-zj2nbd-nbv87cpm7mgj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-zj2nbd/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lqb7d8\" for cluster \"cluster-nightly-4-20-lqb7d8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001181280\u003e: \n failed to create NodePool np-lqb7d8: failed starting nodepool creation \"np-lqb7d8\" for cluster \"cluster-nightly-4-20-lqb7d8\" in resourcegroup=\"rg-nightly-4-20-9g5dwcgvzm8b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-9g5dwcgvzm8b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-lqb7d8/nodePools/np-lqb7d8\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lqb7d8\" for cluster \"cluster-nightly-4-20-lqb7d8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001181280\u003e: \n failed to create NodePool np-lqb7d8: failed starting nodepool creation \"np-lqb7d8\" for cluster \"cluster-nightly-4-20-lqb7d8\" in resourcegroup=\"rg-nightly-4-20-9g5dwcgvzm8b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-9g5dwcgvzm8b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-lqb7d8/nodePools/np-lqb7d8\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"2c73a06cdcacfc08c5ae0807302796c3e3154da70a86dd6ed16c84098e26ab85","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"57f75e9006bbdcb6503c784e4aae06190b496bc4d2463173bd66d696b13fcfb8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"9fb7620a84e0a14940bbc581680a78cbf6f7134a9005afd5f2451b87d1987486","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"ccd3082ac0ec3ee7f5872b88cabf71e9eaf1a7b9a9598367688dc3bb14866a3a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":2,"after_last_push_count":19,"also_in":["DEV"],"trend_counts":[5,5,1,3,1,1,2],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":13,"prior_last_seen_at":"2026-09-20T02:00:37Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bc2d80\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-rdvqqgkbcqkm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a8f31d96-4bf3-4b48-a764-fe35d5af46db\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a8f31d96-4bf3-4b48-a764-fe35d5af46db\",\n \"name\": \"a8f31d96-4bf3-4b48-a764-fe35d5af46db\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T23:51:44.432225393Z\",\n \"endTime\": \"2026-09-24T00:10:47.655792485Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bc2d80\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-rdvqqgkbcqkm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a8f31d96-4bf3-4b48-a764-fe35d5af46db\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a8f31d96-4bf3-4b48-a764-fe35d5af46db\",\n \"name\": \"a8f31d96-4bf3-4b48-a764-fe35d5af46db\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T23:51:44.432225393Z\",\n \"endTime\": \"2026-09-24T00:10:47.655792485Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc002b8f878\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-v52dlnd99fqc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dff53d4-fb68-4baa-8655-9059740a3aee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dff53d4-fb68-4baa-8655-9059740a3aee\",\n \"name\": \"8dff53d4-fb68-4baa-8655-9059740a3aee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T02:19:20.475025137Z\",\n \"endTime\": \"2026-09-23T02:38:20.795241715Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc002b8f878\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-v52dlnd99fqc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dff53d4-fb68-4baa-8655-9059740a3aee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dff53d4-fb68-4baa-8655-9059740a3aee\",\n \"name\": \"8dff53d4-fb68-4baa-8655-9059740a3aee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T02:19:20.475025137Z\",\n \"endTime\": \"2026-09-23T02:38:20.795241715Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"5e96aa58c18568a25ab9acb5502820d47261ecacba6ffd475cef3d142037f291","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"300e775547bf8a76fdcf7fc914a06bfeb3d60330ccc0c5023680429a57300fc7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-mx2sdx5hq5cq/cluster-nightly-5-1-r4tn78 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001788aa0\u003e: \n failed to create HCP cluster cluster-nightly-5-1-r4tn78: failed waiting for cluster=\"cluster-nightly-5-1-r4tn78\" in resourcegroup=\"rg-nightly-5-1-mx2sdx5hq5cq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9e1c75fb-14a3-4cb5-b5d0-9a40f34a6079\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9e1c75fb-14a3-4cb5-b5d0-9a40f34a6079\",\n \"name\": \"9e1c75fb-14a3-4cb5-b5d0-9a40f34a6079\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T23:28:28.748187971Z\",\n \"endTime\": \"2026-09-23T23:57:28.896085734Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-mx2sdx5hq5cq/cluster-nightly-5-1-r4tn78 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001788aa0\u003e: \n failed to create HCP cluster cluster-nightly-5-1-r4tn78: failed waiting for cluster=\"cluster-nightly-5-1-r4tn78\" in resourcegroup=\"rg-nightly-5-1-mx2sdx5hq5cq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9e1c75fb-14a3-4cb5-b5d0-9a40f34a6079\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9e1c75fb-14a3-4cb5-b5d0-9a40f34a6079\",\n \"name\": \"9e1c75fb-14a3-4cb5-b5d0-9a40f34a6079\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-23T23:28:28.748187971Z\",\n \"endTime\": \"2026-09-23T23:57:28.896085734Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"af6920bd4dcef413e75f2c2ee8ed9b6ea9da89a5c3cbc85c4911bd464e6f591d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":8,"trend_counts":[1,1,1,1,1,1,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-20T23:00:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000587ac0\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-f9strbqhzqsp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-f9strbqhzqsp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000587ac0\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-f9strbqhzqsp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-f9strbqhzqsp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"2f40037715e35fc92a35242fe15cfd48e9dba77dbef0411ab7466b2e286500e4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":8,"trend_counts":[1,1,1,1,1,1,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-20T23:00:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000cbd410\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-23-031212, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000cbd410\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-23-031212, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"b7d9ce39ea6386c93e86493f0c036b5ac8e6406a7e67ba9a7045486fddb68882","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1c0eb3626837a0d57ffaef6abf34daac6ebcb26dfffc58a3917d8718d8239c35","failure_pattern":"cluster CA data must contain exactly one PEM block","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["INT"],"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block\nExpected\n \u003c[]uint8 | len:6010, cap:6012\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block\nExpected\n \u003c[]uint8 | len:6010, cap:6012\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"dcdb32a276b85d5762c8f9a4390b250e8e8c154b174fce2166a1284975070064","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":4,"failed_runs":4,"raw_occurrences":224,"matched_occurrences":224,"runs_affected":4},"rows":[{"environment":"stg","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":200,"runs_affected":4,"failed_runs":4,"run_impact_percent":100,"anchor_occurrences":200,"after_last_push_count":683,"trend_counts":[0,52,108,108,108,107,200],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":13,"prior_last_seen_at":"2026-09-20T23:00:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c825a0\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-xxnn2gk7lrb2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-xxnn2gk7lrb2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c825a0\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-xxnn2gk7lrb2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-xxnn2gk7lrb2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:101]: failed to begin creating HCP cluster \"admin-cred-lifecycle-v7kjhv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d905c0\u003e: \n failed starting cluster creation \"admin-cred-lifecycle-v7kjhv\" in resourcegroup=\"admin-credential-lifecycle-test-qws7j9zzgwzk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-qws7j9zzgwzk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-v7kjhv\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:101]: failed to begin creating HCP cluster \"admin-cred-lifecycle-v7kjhv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d905c0\u003e: \n failed starting cluster creation \"admin-cred-lifecycle-v7kjhv\" in resourcegroup=\"admin-credential-lifecycle-test-qws7j9zzgwzk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-qws7j9zzgwzk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-v7kjhv\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000984920\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-zmq2fmksvd6w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-zmq2fmksvd6w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000984920\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-zmq2fmksvd6w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-zmq2fmksvd6w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"0b2e0e2a705eef2a8c34a9cbe9058b556f7e534770a547f7b0609260c02c4bfc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"0d143ad9d871f8761c894631d5ac5ac89f118ac56bd392b9fad3aef7ef29f7cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"103c847c3dc0e0c5a0c1e2041083d44a521ad4942917fb41297142b8bf729ae1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"18a080cede1bb3983c6e834e6eebef5703de52cf674ed437a488782e09deeba7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"2789ba3cd7312b4b858ed6732db313a63af15c5443bb24ef97632db2fab0aa1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"2d2fb06c8452a100942b711accf2add79c08e2ad05176213f3e8e83c23c10eb1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"3c6d3bbf620206422ebf37e0988112e383deeab2de78cc2c3d71820ae7402c1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"44477526f3f5c967017a65f3a9a523c776dd67b3ac4e6dd0403372636e3b1343","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"4c9e4e595392b76de5e57905da765e092ea4b53fb4e4a74a9a478b70b2a59a09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"5783e4c8b0c913d9ac1cfb0f45eb24edd6329f5cb3aaa18343f050038c5e6d71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"5853cc43cf7425055621ffb2ed66038b5546de23d7a240578f0080ea258e2b72","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"5ab72705e757a4285a503be51374c0f65a83934561314f0ab6a40725e9e8423f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"62b8ed0f3c7495d4a962b220108f6cdf7916768f686fc9f71620743b5266548f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"66bb238f691dab83756a60fd4ef63b42d1bcad07037abbae7bfe4492035f5ace","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"6b6231f6a741045d0d4adc8542106388a4942ccd90466630d8b1d4ae0197568e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"760124e7b25aec0cd4af5e7f97d46af297b8834a0ae47c0dcfff744bd3a49ba5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"76eeb452365a9197cccddb41b3c6454114a1f922f7775ab7586fdbf3211fc438","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"7980065f3e3c71ff5248c023d7dea8b862c74b6f0fc9d85d1b0dc6232900a78d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"848a835912eb35c98e5793402aff5be6e85e6150ba7567ff6939266d31765a3c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"8ec3e712fd3e8543d2d9d2a3f6c13734045c633652dd3994ffae220c9f4487ca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"9b3fe187d08170fbc37adc61518c7eeb7b25a95802571b8983f126cabf6fdbc6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"9f4ef32027d9d349f278002e9bcde48f6415631082b03813d234b189e64d7959","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"a3128fe89c2b34a47c935b73ccf2c1a9a2740878a1a3290746b393ecbb354616","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"a5caf3ca78309008f43557f4e7fa4ff03bf651fed81afe60cc0f43daae52ccfb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"b06bc75230ebcfa423f62726ee3b474f1b5c932e5eb57e3f75869553b569f537","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"b3402bd39c1526399877b423af2058f9c4707b63ec0c5f879b3458d0c4b9860c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"b7d497a59299e6705ec5c27d0954476df3b73d478d675970b241ac6bdeb0beb4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"baab246acfa194162e561c3c3cb6c1341eae11f128345f24470a2c72bd2f4aee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"d4bdde14f6cc55c0354fc8f6e82a55421657445ff0aba3cf3df52d2f44eda51b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"d7884279496d93385c3451a48331a1291d1cbd9cd69dab41dcdaacfdd026285a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"df16ba4a6674c679f09b89c1eddcb51a1f9ca845693dcf206c2902823ef7e013","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"df4451d3e29bdc3a64d7d56a21b5e669cdcab64a57c00193734b831648f9892a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"e403149dc0ab41b7958114f780c8dfec643d355ef165e66823690ee143d7dcbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"f3e5b048e00e5607edfcc1021209c6ad811e1b8fec6fd4f78740242a2b4ef145","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"055fc0966640e70d474c53a699b885b63dd8c74064a9e17fb602a8300375a12f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"0cccb5c6694b8a6067983c3b7bf775d7a351d03d11aaae0408425a37126b77b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"15fc45cd3a6e9b691b043a24977de024edb4d4590b2e83405ee977a707b3bfbd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"177d3a6c2c33493e767ecaca9b49e225e0b353fc1f73f3a7f19b8d1d42de2b31","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"1c2d92e2bedb33fe555fec70451124e32fdf767ba423de0c8f828af1bc35cc8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"1dd2c6cc8e4f394a34e0fc7e8213f2ab03bd15a43e139005281c4de11c90ccc3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"1e11434627c662dbbe902599e9237bb9f801d7b594c61e3714ab1bc17eca9161","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"1f20143686241c78e93d2d29986f7163132c10ed802ef6e532c64a395fa4d866","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"2216802953356f2ab624815ca80f12ca32ed7a86476d153301b4ff00b4a54b22","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"22d721cd87f00c16c308d65485d8483188769f0546a555d083b414ba8263727e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"29c1e995b8db1156d89e071660ed02b25db28529121ba2f957571744517c5a69","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"2d025d742d960f65ce0df017beb150ac204ad79db74000677aa6ae4a6d667007","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"310fff41d54de3202e5f9f4c3155139ebbac3cbc401622dae98eb10133d7a4af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"31b6db9373de5fd907ddd06fb828d6f181f6a35afd6c454485818fc463598c5e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"378fe00b23b03f756b8692d9b32ec7d2be8def98bef3a4def31f682ef75a69e8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"3fc7356f7fe16e5e609d0ba5038de5a411cf304f107f57ef9c728c54c87f3726","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"4128486c577bef73cb4bbdeb9e75cb7a1aa87e84c86bf43283309add2a1f3286","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"41eb5f994f3ddf34344ce72d61df3f563d8898b4a6450b02169b782482eea506","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"422f4827576f5663abf88da2acea909e13a8a93903be9e159726618aafddf3df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"460443b8e000eb1965f80fb4652248f0dd60ecdb22f5297b96d6782ac13c0d1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"47aecd3202086479a3f88dfa9da71e860867c259c29f1b085fdff23422a0e308","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"50239d603bb780e2e003b8ea21c9046e3218d0cfe34261f655d2951ccb605eb8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"6734cdcb4d8aef16e200ba287bf768f9fdd46715bf7a03214569d57b945db659","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"67ea86669999abbb4487fef31f427939e9e272ebfccd1ab225343a4011bef471","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"6f1244f44ecd8b4386870618b25b694425b1ab47b331ebc35c4fe4bd67d4a631","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"7921192492632722db01f265f3803c9c63f3df67888105c25cde9c26f19b4bb4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"837390542bdc443991e02ac2fc910c99edc2dbc38f20395d7af528a232fc43e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"85546bd52d8389ea58add506b472388cf04ba0e5467d3dc4d5f9a09d41818886","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"86320aedbaa0e91f0fa200e2296801441f3c0f977b35ca4e9212e9d6e4951e15","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"88fdaade50cf0d1f17b365e4c98e2ce7167b37da529946fb1b6ce451043f8534","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"8ebb44433de75ae1a59b237d32db8b6690cca6192875ac16af87e54b2f413ceb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"93293e6b4fe69ff746741e539bee57e0ae0499574cb09083e0dab652c3f3b297","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"9c9ed5a78aed1b362d5aeec8761fb5cebdedeafacf061840b8e80db9cf7b0faf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"9f7fbf5b4c12c9689dde71f7903041c031a8c2c9d7c2349fffe2cf45bc4705b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"a068ba3d18cd23c9aba119652965396d00f35a6fa23cfaacf2c15e2793881de7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"a597ae1c8f906eafb58886f6816c5618cf09b5cf8ed7636f9cb0360f165e8bed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"ab4cbe036556b338289f7725734bbb513d944dccdc26d6efeb46fec80dfa5a8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"b64a093bb316ed23148a238480d3c3b297377eec6a0d5b6477f4d2ab1d28f450","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"bd7a905f012148451aac90fdee9e8bce962b34e00b90af1ab067a26e5e89f381","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"be9719f4635bc456af0045a5b461c223ac5096f83cf7ae4dfc406fd8d8ea51ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"c623dc307c56fc7aabf82a0862f91773eb11d3a0736caf6f633dfbc71a33cbd4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"c70dc9b2344a22c84855927d709917851252038a14a65b1b42ef8df388ce4408","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"d4353e2de7ecccab93779b4bcfa87a57824ecdd3130b162614c71c910af28701","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"d9b4da1e3652b9dbb42131926eb40a3f68dc2918cb70eb3c61df45ff7270e18f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"d9d382c4357a2f83dd034a43e9ce78fdc4093d1bcef932b3734fb4e2aa054e47","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"e0c1ab0a923b3315a04331d9fd43f31fc3b87e0ba01d6d856f153ac9ab08804f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"e6c51f65ab36666b3dcc2d0da28446428775135469fa57e5d425afa8e424a2a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"e707ce73c3444549cb1b45692d1a7984bcddc94e2b321c8ef6f32f988e83b46a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"ed35adaa8e06ba709899bb9363d50812ad914c59c4485a7dedf2f384e127f581","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"ee1260742cfd6d8b1b7904c0717f61165015ad94e818f219111800249f781b4e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"ef11ab2cf959392657372fa0909fc2c548315a7d685b35f76171056b4abcc194","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"fd4fd6cc3f3b5dcc77f5c5745a3117eaf3d30f9f2b15b40a36c3cae3ffdee395","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"03b6a7c43565f4d6457ac2a12a167a7c246d083a848784af01b179d01a1b1959","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"08271482dccdc39bddd2cb376b98b288f0d159b3c06e543fde79c1ed7f9713d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"0991f03a2ec81a59452280bcd4e0f4b5e4ae53001ecb34846994d6f30417c74b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"10750c53b0c1dc3c380bfb98ac1ab6ed054bc9fb7563fe02eae47c73ddc7789a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"112783ba7b95c403dd3d93efc5449c10448c7bec0d1079842d6a9d439cbc44d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"11d8be1ddc0e0a10220e86111161442393e8e4fe9d7bc0f56690967982734e25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"1861c336e850ec720d54ed12538396a5968d46ac38640e35bbd2462f646633a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"25c4e000866ba7943d0ff38f96d5187f3e10e9481cdf357cce05365f644f12c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"25fa332b34d3d0c6a3770683fdb0f70e3a425bd1ddea7417e681b87d353bf917","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"274d98c050c02ba40012737f76ea63998befa2394a42cec617bc7a856ad7682a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"2d9c7e2f49c360d0ece72f03034d96b551f5b6fc5126e16052ad10f272e652b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"2e22a4a575095366d9288929a006514acfaf8691e9e3d7d3e79cb23cd3ab9b93","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"2e7c66dd5b354188df699a79024b9c9df5f0658bb90ca1867f9faee1ea20cf9f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"30a0cd932f16b4934c6c04f40a2551a1a8dfd6f1c7e9b6c8a18ef85340a6c9fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"319cb259eb78c0d1bf8cdf9d60b591524a4d613b7fa239d634ac6dac397f50f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"4983315498d70cb31e76f7699a313650db35ece01cd779a279712dcb8442887b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"4ae18b7f4aa0dc6f2688473720e3fac39e9593abc7a2693fad640b4439398de5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"53e43e1b5eb0e6100a4da856364350993487635e225b11af657e0f6c38671ad0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"56aa8b7d41bd143dfe4120e95fd6d7e4c9bbd66a4330de0e627de46d6435247b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"681de8d8b7da047de7080e050ef511cddcd03a883076ac9fc82e103e6deffdcc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"6c40a656c38a297b57950efbb351560b8611e7673ba986f528cd6c276410f7a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"743a5ab4bbb37dae80df81db4c44a32cb21bb079217ca716a1aef7fb333ab9c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"74c7a39324f13af264582228997f17104143bff25f38bbc535fb0f57457f5572","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"75ef17175d53859e312f86a0739d2f6b44c2045f9da4b4c135abca826451b8ca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"77100504840186e9b5fee4b7540e7daea129e844b89dc8b99d371650c4f68531","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"860aff23781cffe752cf788549eb40de12d922321ece7a067ab30debae897099","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"8637b2f6ba5650241aaaaa116a2282cef63f77b6b74df2a63cb50451ef5af63d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"90e2f9b7aba95bedd6261d87467024ea908cc72b001a451c8f70266a78eb279a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"91d6b2290275c146a985f2956ee1842c1adbe4cb3327ef55f9f3f2426b0667d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"933c56da08d65cda2aa6845c5dc6f3d13c9baf8e9bf94638891a75c18cc282db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"96b255a8cd6cf44fdebf8d73507be6fe889f89f698f8b5be8e30089737a7dc45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"97788927dec662d1d0e17e084bc25e4afd04ae188871b50b8331a02a8e1512be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"9ebaebedfaf6975ff95a4970a8f1bff72b03b8677d3a8aed73410a5bdba9d566","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"a1b4c80ff80950b4b4b75a12d415c5927a2f10d62f1b79a4d8901849717eafe4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"a70aa82f421e4acdb041ad16496c250af687badd0c3ab467c9c3a946de950fc2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"aaf98359994aa5a99a180a2d9a07c8e60eeb104fc6de4939bffabcaed04533c6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"acba9062b228abb646f0453b339381dd60f3095dcd2638eec0023c8404f155f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"b5cc2d0a8fce4dd78b8efcb7b33bfb25bfc5ca2e9fa45fefeb729b74cc3d9a91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"b6e080a5fbaf018893bc01e662e627acbbdae0f7292faf237f307e3db8105b47","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"bb36fc8d69b4ccfcf89bcd6519a5915b66e79f46ab6877afd7421eb5b01710fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"bdb9550e9115080d784a804bd1ded215f1f23126748ec7f4f9a2e0e56e195c67","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"c13d02c7d475af09cbae9d547b5a17ff9551e2bd6740d79a99ba3a7e492e2feb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"c207cf1d189d0d3cf54778aacedacfba7248221d8883dbb57f709247ee506b89","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"c4cbde7cb1452b4cf73b0cd9d594e432b97d6987d73cde5510ab0cc87df2fec5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"c89b3ce1fd98fb962801441940537ea6f73edd87b76bd04625e9c99544707285","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"ca6b9a69cdcb8d678a64cc522c2eded46cc7048e98db8b60701be431f4451559","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"cab56619bdf677848c93b5b8420ef2c26d700e1764280f4910b1d4ffa64fd926","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"cc7a5cd5ee8206e6d20ce4a924f1368feeab1325081fef357d5c935b3b9d6af4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"d00bb74f4e653fe5a51ef79bdc35230c19f456d7de25ee574b5724ead42ab6f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"d2bb6e2d403656c49ab04e83c3bb7e2ff4b986e11e9b4058682e200cf346b8b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"d2c0b34854603aac3e86bd85677fb864f529bab16a93499e7d50cd286c96374b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"deaac275b010033e9f5d3593bc261037d301f4d2a1bc4467c6ed5a8265146456","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"e52bb7c8b8702e4f5f0f12d35371440fa26f8e7305b750f17f3780409d1cea4b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"e7aa44e9bbee4b315fd192b4e7f53f081c938e5e5502ae7ad2f73d753bd1addc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"e913588f2e94836d7f113c48b7e55eb49e70db7bd51ad09a6a1f974c6f20970f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"f036658b7e69c80c317d2d24a945358af0e8f394a6c34d18d961e843159d8f04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"f986660f71fd5e2eff4c2a2d31f4185e824ad6773c3a62840cd61cdbf74954e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"fc9cf6cd8f627b96267ce3a2d2df20ee00f7becd9a6bfddebea3bba60248db33","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"05e00ea7e8fca29e91df7921854d0045d7d268a32fd5b1fb898fc496d1176180","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"11d5639b580236cb0256012436dce0eb857fee45e4e28bbc667bf91a1e2a0356","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"14a5ec07af57bb86a839a52a1b638f1fd866b70c11189aedcf0432ac10bfe09b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"16a3295488b90d396bf7321da82f4d92b00ef4b32d39dce5c38e9923b96dbc4c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"17df3d13f5564cb9f511516ded4286c659befbe31426e6104494fef23a28c7dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"2151fe66fba30fc7b88c2c51c44e694bfe080433a2c89a3e8c0231bf9e722eaf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"23635ac418e137f892faf70ba7be1206ece8952522444085c8aaa3f5478c3c9f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"2404a21cae14e1faee7ca7136c2409c3fd7e7c4f15c1dc45b8b5f77f87c756de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"289eec3a6e696ef46ddf1c7f86dfde24b915efbf8c0d349295b1e0b4238e5bb7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"293200cbe5510f7bd2d0f01fa85b6491a31df5a39073a66ae7eb7c5c4fada6ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"39ce6c36b21ff03c35642644eded4e8649bd345881f2acc5adac6b656f377e15","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"3e647150f33e3ffc56d2c1571c807f82cff11cc470178b3156913149e09c5fa8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"480cbc0b92eb45b732e39ec5305ca8fd33c7a9199ad3f175f39e7864760ec4fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"4af054ec58b34153ea3601f9333faa1564f06c8e2e087fdd7e3bd3772cfac636","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"4cc6e4155ae2365f53ebd0b1ffad135c6f6a236f7ef22d5a24dbc10df7bdd269","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"5364033772494115e4b5f5e40cf01432156a12589305922ac4726f437c03dae2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"55745c23236c64509e7a4aa08f812f9a02fb58ccdc1c1d4ecfbcb51cf8a27003","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"56826cf4f16e45e7487b87b2af3d3c6ae3a0e3cfa1f3cc7fa5babe12c95f8be4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"5a6930dc2b2b119484f09428ee05bc3e9aecd5bc4212a7db9f2b1c6cb97736a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"5afa96be6ec41f79a3fe07b706176526c35b8138b5810b57ec811f7943de2343","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"5c4cc073f23d1d65b6648dcd7501f81e8e79d5adb2e4adf87d81a8923986aac3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"5c703ec8433f62e4f8db39dbfd922f3949682c8caf1f3cf28f93dfdf421976dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"60b4f8f14ba59be897db2077b4c610fcc0496c981276a321ae544895b832ed97","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"61db7bb228bffe819303eb1c357004a6e589362fbc7f05ce28fb3a3517ae14d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"70e94d44bbd3c7bd987ab03782491c93ccc9a29d5f05ba7e1fe547fb16dfc8c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"73468874c7c9cf8732ad731b3ea78e3133f91cd177d1caf52e6b177a524605a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"7a99d6719a1f3f880e142b01cf1409fe5b22d96c4ae9d411a7727111fa56ca3f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"8790ec9b27ab1f793c595a203edf5c900eb4a06cb5959f9f40e481ce25b3a61a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"897c8759b2596cfd877d3de8c8f27dffa79f5a10c27cc01aff9f1f90af11382b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"89bd7a623adb5354a5f8c65504e30ec84efdd85ed2a46a72c2401b213c912802","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"8b94f2133755206c3dd7834edbf78fd4d7dc593a42786d5b82b69feeefc4db0c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"8d905c196618d4530f22d1430069e41b0f742c52c337188e2d26d0f1afe42e3b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"9168c97144a1d17cd7592e9a7ab07318188e5559433bd9a9095c6df99462ecc6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"9de186bca5f6094a9f6224b75e41681f8fa5fbeb6c2779f1896ba6003e742fee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"a016b5b38d2a4d75d53b6bac598237104a6247cc3099e70e2994bd0e3e4f3c67","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"a47196edb9e16f5059f4721a7f42f6ff2b4d49de1c50e00267532d5dcea13e25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"a6772d6c106905eb9f6a3066eec528ee5954761eb637001321b4f9f88fd5856b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"ab318d0dddcdf34758c7bdb0bc2d5ce297d508c1f6dd190886b2c9185e339a90","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"abd5b0bde20929e08f3cff92d0e75df98245e589a0b828e32c756457f0299f12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"ae1744745976b65d4485f1254ff1f7de0714d5bef862787f9789928eb0679c2b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"b0a448cb87c1db6abd793ac1e2397d859df5b317704c9daa67d5658b46c0d87a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"b1da5cdf26e0eee3ab2b8e8be44d08620bdf20f8433bcd360f87f3dfcf2f9c34","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"b4a5a50af6009581dc90fafaa87f4ddba1bd3e00165b7e289232ffb8b1070898","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"c5e0c7d19a8e2736fe9718b51962cb6ca433e6e88b4677be6a3d840fdf36b0ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"c67f5535df3dc3502025fcc409699feb2bfc36c4adc644e0b97406a0edaab71d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"cac1cd023c3da045c4f4887f2d690e9268981477403935ba03e92dca7a280d1d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"d09e517a2b65e275aa9848c4e08fda1f72ad388ed8491a5735bb912f6ee48eab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"d0d4aed9b08ccbcf282b2f7c430431f065a7afdf35b7962e9f9e39791686a77b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"e5745b7eb179164bba7685d4463315e3b097f4a638a982a2fd442bba90e4bc17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"e731596e866a4f1aa8f9bf3e441194ce1f7bc0658b366cc2e9ea533c0b061f8a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"ecae1b0e2c90b770a76b856b5e8af7e32770584b190feb7e81bff82bfbf789f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"f14e289d6a4ebfe3fb35f46d7570468b025aa7a11dd277e640517937dbc0a918","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"f3a544a747d524e7c42a166bea6da176cfc9fd44cafd865f6ef95fd355051d3b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"f8270308b8276ecf0eab70cb17d92106639dd7da4b191567abdb0119310f33b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"fb2833678dadd3afaa201320b7355bd70292569e1299718dce7b9367cb5b7866","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"fdc3dd15ff2799efe3a6bd04abb34c603047b1aaf5a9d25deb018df7ea20395c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":24,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":24,"after_last_push_count":24,"trend_counts":[0,0,0,0,0,0,24],"trend_range":"2026-09-17..2026-09-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:570]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001026280\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-downgrade-shn7fn-zvxvfc\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-downgrade-shn7fn-zvxvfc/operationStatuses/08584113985869276090\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-shn7fn-psrg7w9f9xgl/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ev3yo2onirsd6\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:570]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001026280\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-downgrade-shn7fn-zvxvfc\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-downgrade-shn7fn-zvxvfc/operationStatuses/08584113985869276090\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-shn7fn-psrg7w9f9xgl/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ev3yo2onirsd6\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:71]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001180e80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-delete-hcp-cluster-rdm6f7\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-delete-hcp-cluster-rdm6f7/operationStatuses/08584113987041544035\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-05/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-pxh9tsgnpxwk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rcpnvqrcxdqmo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:71]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001180e80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-delete-hcp-cluster-rdm6f7\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-delete-hcp-cluster-rdm6f7/operationStatuses/08584113987041544035\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-05/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-pxh9tsgnpxwk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rcpnvqrcxdqmo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001e4b80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-42bbcr-vkf8rz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-42bbcr-vkf8rz/operationStatuses/08584113986227757596\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-42bbcr-2rk6t6mlf2lm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-pgend26hbad42\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001e4b80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-42bbcr-vkf8rz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-42bbcr-vkf8rz/operationStatuses/08584113986227757596\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-42bbcr-2rk6t6mlf2lm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-pgend26hbad42\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"00d3d6c70857d58cc7a77ad3cbf60f9749e64131ae86e9c5bbed23a03414b07f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"04f03534a2c11433842c187808432c4fe56b5e127cb74b63d712bb47709a5195","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"147bc66a9ffa2ef0eb84aeffac927813f9dbc0de655e0de415f62f1152f420b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"1658b955eb3643def75168d8be5b7cdd2bbbdffba30215221b837273eef86550","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"207076026b7204986c55fc7402443d46f35a2e59e1d8abc866ccf5d15030f36a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"2577d97b4cfb09d0e29fa89cfbebf02bbe5521e4fedc6b8305c75d3a21b78bd1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"29b4f2a9dcd277b5f12403d713f2be8b0efe8f05e42b48c6fc64c2d68594b63d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"5743f1054d0ef1e157783ab637285ae718a794a73ec3f9d8e1cd66e8dc185998","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"578200e0d6235a987906427ba8a60a1f9bad1525471deb0bda8be5707a26c15a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"5e161d805837a94ed6359e3e20abe23d9621b72c256eeb4160963d2f2ded8efa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"5ebbffe49b3cac6c858179fca1ea96cf96ef5a9cb1c80ecbbe6a096d6439ff4a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"711a23fdc6554e3f11a83326688aeff48062de542e6bf7ed6bf7c7bb3c91aa4b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"72b7cdf642ce3d305cfdf2201f3999ecc6881553d5e2167443ac80f1a5e34e81","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"8a3707252faaf14736b2be9fdc31fbfb7ee6fd9a2c628081309edfdac8fa18a3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"a243861fdf0dc90f346c36654a0f53ec596c9df69556eca343abdfd26653e306","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"a96addc801edde278066d55645bd6687bd80418d4c10c554605fa06785f260d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"ad868390107afef487eb4050fa0c3e0b12550c6ce4d10603b345f5c954ed7fb6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"ad8d9c518342d140737ad5d4a5fabe301705900959b3e86d1f3308ce70e5f962","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"b5d93b8bde9091dbdbd0277f382eef3ce49ab826cf8e62f6b226b70f017f85a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"c2e9c1eb22fec11e3787804927eeed16352b79cdf1428605ce93e959bffaae09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"d2c063f2267dd2c0079f8d444aad43b12dcd51b7cb090dcd5fcfa03d5d526369","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"d4b1caf980e450681931897e537f4227ce5d0899055d23d5f616842e4e73d6eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"e0d27c79c5899c123b80405656642920e76f42cb9309a89bffa6098039e41580","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"fc7baf411234e091570207b597ac8f8d793f9adeefa4cf31f1037c197fa10337","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}