{"meta":{"start_date":"2026-09-10","end_date":"2026-09-16","start_at":"2026-09-10T00:00:00Z","end_at":"2026-09-17T00:00:00Z","timezone":"UTC","generated_at":"2026-10-14T12:18:59Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":751,"failed_runs":632,"raw_occurrences":873,"matched_occurrences":343,"runs_affected":102},"rows":[{"environment":"dev","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":71,"runs_affected":27,"failed_runs":27,"run_impact_percent":3.5952063914780292,"anchor_occurrences":71,"after_last_push_count":17,"trend_counts":[0,0,0,0,0,0,0,71],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":27,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":7},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fcc7a0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-jbw245668lwj\" to finish creating: GET https://rp.j7559936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d2a250e-0bac-4f44-a708-541cb00c701f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d2a250e-0bac-4f44-a708-541cb00c701f\",\n      \"name\": \"6d2a250e-0bac-4f44-a708-541cb00c701f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T23:34:27.803730917Z\",\n      \"endTime\": \"2026-09-17T00:03:30.376925674Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fcc7a0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-jbw245668lwj\" to finish creating: GET https://rp.j7559936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d2a250e-0bac-4f44-a708-541cb00c701f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d2a250e-0bac-4f44-a708-541cb00c701f\",\n      \"name\": \"6d2a250e-0bac-4f44-a708-541cb00c701f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T23:34:27.803730917Z\",\n      \"endTime\": \"2026-09-17T00:03:30.376925674Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-v9lkhvnzdhld/cluster-candidate-4-21-jrkc8h should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ba8a40\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-jrkc8h: failed waiting for cluster=\"cluster-candidate-4-21-jrkc8h\" in resourcegroup=\"rg-candidate-4-21-v9lkhvnzdhld\" to finish creating: GET https://rp.j8160640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61166157-eac2-4677-a0c1-58bf48e34b38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61166157-eac2-4677-a0c1-58bf48e34b38\",\n      \"name\": \"61166157-eac2-4677-a0c1-58bf48e34b38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T23:26:49.985549857Z\",\n      \"endTime\": \"2026-09-16T23:55:49.56140206Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-v9lkhvnzdhld/cluster-candidate-4-21-jrkc8h should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ba8a40\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-jrkc8h: failed waiting for cluster=\"cluster-candidate-4-21-jrkc8h\" in resourcegroup=\"rg-candidate-4-21-v9lkhvnzdhld\" to finish creating: GET https://rp.j8160640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61166157-eac2-4677-a0c1-58bf48e34b38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61166157-eac2-4677-a0c1-58bf48e34b38\",\n      \"name\": \"61166157-eac2-4677-a0c1-58bf48e34b38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T23:26:49.985549857Z\",\n      \"endTime\": \"2026-09-16T23:55:49.56140206Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f12a00\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-mt67hxmh7jr7\" to finish creating: GET https://rp.j7063936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2d6116a-8dc5-497c-a1b9-37dbf3d9da56\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2d6116a-8dc5-497c-a1b9-37dbf3d9da56\",\n      \"name\": \"f2d6116a-8dc5-497c-a1b9-37dbf3d9da56\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T22:42:15.998420325Z\",\n      \"endTime\": \"2026-09-16T23:11:18.627710941Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m7s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f12a00\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-mt67hxmh7jr7\" to finish creating: GET https://rp.j7063936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2d6116a-8dc5-497c-a1b9-37dbf3d9da56\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f2d6116a-8dc5-497c-a1b9-37dbf3d9da56\",\n      \"name\": \"f2d6116a-8dc5-497c-a1b9-37dbf3d9da56\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T22:42:15.998420325Z\",\n      \"endTime\": \"2026-09-16T23:11:18.627710941Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m7s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100354655547559936","occurred_at":"2026-09-16T22:42:23Z","signature_id":"f2b94e7451cbc051fbf0de3b84883a92be81c7f11c25cc3c65ce003806a07176","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351996258160640","occurred_at":"2026-09-16T22:31:53Z","signature_id":"6606b623ae18c88ac8a9b2ec7e9a18026cd7fd696af00a64c201797796b23884","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100346022097063936","occurred_at":"2026-09-16T22:08:08Z","signature_id":"40918e34615df9a134d5bd8ce9ad3cec41e9fc04a925911878e36407f747a316","pr_number":4878,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100346022097063936","occurred_at":"2026-09-16T22:08:08Z","signature_id":"be79ce9f9e2a3c3c1a4622c360ca3519703f1be9bdd39e307bbb6ae587836434","pr_number":4878,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100344321164185600","occurred_at":"2026-09-16T22:01:23Z","signature_id":"00790225a1ec5d2f2111abfe85ea313169365d1041de24fbe4fa745473e8e330","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"0f792fe0731eefcb63f38f15bca9a1aa450cefba466c1d19cff0ae8ed18fb263","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"2698bc1670e697e5f02bc62eaeda2db20989111b43c6c7fcc9af92e68d70c247","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"3728e9e0a9b25fc2b96787f8795446de81019e99ef3c6a67f320e9809b7bdae3","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"50dab7e83b7442f97fb4586fd719c0b5802925b2f02cc972849692995a093dd0","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"79d660f1bacf21e2718e2b4460bbd751ebcedbde1fed3611a2e8b06be51c9b26","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"8d9c0886049dce2e52d531fad148d6ed84eeb2446b9625587b295b408c739bc3","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"a52a7332954a9dce3940383eb5861ec4596c82a180f58ab40695aaa3094465e8","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"fc5496eac86ff2a1981e1d0213c04f2b8c1f98c1155641ca8fa469251820f45a","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"12b8d48d4627f15facb797beba81c6d3c5baa8cff824ed1576ee1a132af8dff4","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"27af66bf8605d41a0eed60544affb038a78040a481edac1756ed6ae13ca1bce6","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"6d4635ee801aeb366b63d020c6eeb3ed381519a10bc1a02c8e00a5d443d6550e","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"e798e0c9ad920ab5c7e8728fb7d75c823add0086ed40ac0f3ccf60ed14187ffe","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100299874200195072","occurred_at":"2026-09-16T19:04:46Z","signature_id":"5264dd1fe40eb5bea2fb2d3c2020b90fcd8c767922fd709e8151a6988e36b5d7","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100294017282478080","occurred_at":"2026-09-16T18:41:29Z","signature_id":"5b5a6e4f2745283c541cfa867e89616d0e7a33b663b83015979c872710cb256e","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","signature_id":"131d9678ec571fa272cfc7362879effebfa9b5e73d92d8d4799c9e8c4dbac244","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100262131709513728","occurred_at":"2026-09-16T16:34:47Z","signature_id":"4d2c343c16f83436cc3986db89b356fdfb3e28de9ebaa9ceda587e5572a6ffc6","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100262131709513728","occurred_at":"2026-09-16T16:34:47Z","signature_id":"7516a34fe822a219e86822cf9a1b8d51e620d585523f3da6508b9fb567e9b286","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252533153861632","occurred_at":"2026-09-16T15:56:39Z","signature_id":"8fab9814450d28b4f592dfdb0cd8c2267f5fe8d7043a907d74c9a720713cb056","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252533153861632","occurred_at":"2026-09-16T15:56:39Z","signature_id":"d2849a0a17cc4329034ec17d0d9ec1983dfbaf5324ca580cd9753b318ae0a9bc","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"51205dcbc010a62ffa53522b05b81b83d3e49f95a801c1ca50c89cf5aa533563","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"7b40ea74a666a71f36b6120544e0bab8b70aaea87e5e9b851b3d3cee913ad9be","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"8553584f5882f3e24dad240bf62c9d51df1d078a5dc179dc15818be725a00b81","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"9bdd91449d29da7a6a1036ded0dff71e7cee7dca6d3b72ee155f7272dbc7ec9e","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"f0391e98a8d1c457ea51b54726e3c7af785fdf41b5e338fe271c88dbc805d911","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"52b66be1d91cf6aa4302f446e0cb6737cf15f6d2f47af03effef5a56fd561fbd","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"53f4dbcf0527ee85fd51e8c2f839babdd74e956b703ba4e02f72dc66ca32e8b9","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"593263a65b95e6f760aa7210a803a4f621c8333076597839a59808dbbf66d1f5","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"92ab9e016befa8294f948260d25c72e7a1d80889506782e1f2a4bb4fd849697e","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"de1bc95acf9d163b0a4c5d87ba6c29060b3bb4b653bb97edd817c695a7ef262b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"eb1186f89f1efbd9f2fb8485a07c81a2445ab57e2c0781fc90a1717593153377","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100228663373991936","occurred_at":"2026-09-16T14:21:48Z","signature_id":"27221439d7603c27de714a5d301e36e3ab9f94a39958b4fd4679cc00d417d51d","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100227428122103808","occurred_at":"2026-09-16T14:16:53Z","signature_id":"0240604425a3af254fe1ce6e815c04f07285897c6355ead3254c138853f560f1","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"416d3122655e04bc6a1e6cd87fa70c5433f7bca4d0212f66aecf79af52b1d0a5","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"42ea74c98dd160984e4c829f799e27d45bf2b5018f73c2238f76e9d8d6a3f703","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"4634c0871869430762696feaab7758bce3c5d6614cf0816c70b2c3b8b4d3ac0d","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"773504a23a339bedaebd473b20c28b68aef28fa8120eb503e3c2ada5990e4cf7","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"97a8d855655aa481b47156e39fc63195c1bf1486f4d7c41cbeb84f0001f75fe9","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"d1ada8baec3cfc5a8db7ecb4d82edd81b7f41fd13101da84520ce2bb5cd103d1","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"3b8fdb12977219a6ee2876ab164d9d979ba1367fe5e61c35ac3016ab87e10d51","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100213017273700352","occurred_at":"2026-09-16T13:19:37Z","signature_id":"1e90b34a567db73b55b0b0389a5bd30033829227bcd0152b9cc1ea0cbbb4ce6a","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100213017273700352","occurred_at":"2026-09-16T13:19:37Z","signature_id":"212391e614ece2195740bc51838047e93b2e7ba13e0b53f412f14fba63b492c3","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100196783199096832","occurred_at":"2026-09-16T12:15:07Z","signature_id":"c6e0c1f0e5cc2283be488e36d90c53cb7425625d072448ae938150a9f6555ec0","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100196783199096832","occurred_at":"2026-09-16T12:15:07Z","signature_id":"d49dc00210e1aa19acac371c4b3bbf0262ff2f7023f27f607ca12332c134c5be","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"3bf8a07ef9de6057d5469a6d5677df4989f7484e06e3a81c4e3df9504d3c4305","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"3ec1682204cd21163b91258f7fd79ed3ef2df2235e522890ff723136f72137a4","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"5d3daeaf2541170cf2048425db2ad55ef8977c82724f199e4513a26eb43628d7","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"cedab3b9f607c3ec7848c570f240cf2b35352afbc688b78a3a96a96f564deb7f","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","signature_id":"40e2d334636405f418ffb52c5061fbf30366656d327d9d8fc83b6e9d9afd9837","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","signature_id":"5b34bf27b7432e062ab59dae95c6e2c2ce5726c184df3b6d5751452cf7c1aac8","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","signature_id":"7fe3931d094682c8af6a0a3a715b07aac46854b46c2bbb9f6ad09b29f24ea4b6","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","signature_id":"c5a888e0079b8b0888371862a2ca292117eb1766fc5723aec9962ec739421dc6","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161384225443840","occurred_at":"2026-09-16T09:54:27Z","signature_id":"3fd557fa1356d64ec7477ff53929400119de4eda55d74d194fe4734c675f465b","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161384225443840","occurred_at":"2026-09-16T09:54:27Z","signature_id":"49e9dc4352795e52ee09109b3c354fc6854bce4fbeed1fea1be113f6a6209aa5","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100153419804184576","occurred_at":"2026-09-16T09:22:48Z","signature_id":"1df8582823fde891257155971a259af217e7163cb77525129dce1bd9c178bb48","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","signature_id":"5a06bfe3d19814f7c0a59d9e13f30fd4c6f1a6a7313da746fc7cd3c44a2f7939","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","signature_id":"5aa313310cb85b534b6ae9ece7e869257889473057f8f6e20312e1cf45aa795f","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","signature_id":"8fc547e896d5e55d734d3ddd67ff9ad51e9b0658292885c96f9396cffb7b85a2","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","signature_id":"b19cd1523bca64c96d563324ee6475a23c1199cf8856040f6afdad3121dbc86f","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100147953778823168","occurred_at":"2026-09-16T09:00:43Z","signature_id":"d15fe82d34504ca3d38710b46e1d54bc220f9fcfedc628bca6bbc3cb0cd4dd6a","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"3f36db93067dda2082a5ba2aa4539a20530e919b951302455f99df670738d8ff","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"94776e25785ac46d0a71172bc6722ed5b40d2bea8cb9370d89cdf09f1fa5c805","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"ab25cdf796ca0e23837f6fcc81a18f44c0e9d3bfb187f38beeb217788731b679","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"1f776ef61b45ee091fed66db65a127929ae7ae7228c6ef8a3304e727c23cbe5d","pr_number":6879,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"9f89db0ec4b8c3901dcf3ea1dec0354567233659a0ca459f22959ae94e89dbca","pr_number":6879,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100119365197238272","occurred_at":"2026-09-16T07:07:29Z","signature_id":"8485b82a0c9f334d47923eba74cfc4cebce6914810909ffc70f3acbe8579b2bf","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100119365197238272","occurred_at":"2026-09-16T07:07:29Z","signature_id":"f6b4ca993a39a3f8a63f6cce0ebaa94328998c8c08a83ab0da84bb108e61207b","pr_number":6939,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":46,"runs_affected":46,"failed_runs":46,"run_impact_percent":6.125166444740346,"anchor_occurrences":46,"after_last_push_count":23,"trend_counts":[0,0,0,0,0,0,0,46],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":46,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":46}],"full_error_samples":["alert fired 12 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T02:14:20Z\n  Ended:       2026-09-17T02:34:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-kv8ld\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-kv8ld (namespace arobit) on cluster ci01-j7494016-svc has averaged 177.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T02:14:20Z\n  Ended:       2026-09-17T02:32:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7b8f98bb8d-xxtkh\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container mgmt-agent-controller in pod mgmt-agent-7b8f98bb8d-xxtkh (namespace mgmt-agent) on cluster ci01-j7494016-mgmt-1 has averaged 189.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T02:14:20Z\n  Ended:       2026-09-17T02:40:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-8987ccb77-nmgwd\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-8987ccb77-nmgwd (namespace clusters-service) on cluster ci01-j7494016-svc has averaged 256.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-17T02:14:20Z\n  Ended:       2026-09-17T02:29:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7b8f98bb8d-l5m85\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container mgmt-agent-controller in pod mgmt-agent-7b8f98bb8d-l5m85 (namespace mgmt-agent) on cluster ci01-j7494016-mgmt-2 has averaged 180% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-17T02:14:20Z\n  Ended:       2026-09-17T02:33:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-hmnhn\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-hmnhn (namespace arobit) on cluster ci01-j7494016-mgmt-1 has averaged 217.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-17T02:14:20Z\n  Ended:       2026-09-17T02:29:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-d9thl\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-d9thl (namespace arobit) on cluster ci01-j7494016-mgmt-2 has averaged 167.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T02:14:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-85694f5cf9-96x49\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container aro-hcp-backend in pod aro-hcp-backend-85694f5cf9-96x49 (namespace aro-hcp) on cluster ci01-j7494016-svc has averaged 418.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-17T02:14:20Z\n  Ended:       2026-09-17T02:40:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-8987ccb77-btgnn\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-8987ccb77-btgnn (namespace clusters-service) on cluster ci01-j7494016-svc has averaged 253.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-17T02:14:20Z\n  Ended:       2026-09-17T02:30:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-w55jk\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-w55jk (namespace arobit) on cluster ci01-j7494016-mgmt-1 has averaged 147.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-17T02:14:20Z\n  Ended:       2026-09-17T02:27:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-bwg62\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-bwg62 (namespace arobit) on cluster ci01-j7494016-svc has averaged 130.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T02:14:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-8987ccb77-sfcv9\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-8987ccb77-sfcv9 (namespace clusters-service) on cluster ci01-j7494016-svc has averaged 850.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-17T02:14:20Z\n  Ended:       2026-09-17T02:33:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7494016-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-v5ccx\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-v5ccx (namespace arobit) on cluster ci01-j7494016-mgmt-1 has averaged 160.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T00:07:06Z\n  Ended:       2026-09-17T00:23:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7559936-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-554b597df9-2phx7\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container mgmt-agent-controller in pod mgmt-agent-554b597df9-2phx7 (namespace mgmt-agent) on cluster ci01-j7559936-mgmt-2 has averaged 182% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T00:07:06Z\n  Ended:       2026-09-17T00:26:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7559936-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-7ddmw\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-7ddmw (namespace arobit) on cluster ci01-j7559936-mgmt-2 has averaged 200.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","alert fired 6 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T00:03:50Z\n  Ended:       2026-09-17T00:10:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3332864-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-2pvnx\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-2pvnx (namespace arobit) on cluster ci01-j3332864-mgmt-1 has averaged 186.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T00:03:50Z\n  Ended:       2026-09-17T00:10:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3332864-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-dd7947c67-r26c4\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container mgmt-agent-controller in pod mgmt-agent-dd7947c67-r26c4 (namespace mgmt-agent) on cluster ci01-j3332864-mgmt-1 has averaged 191.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T00:03:50Z\n  Ended:       2026-09-17T00:13:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3332864-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-dd85c69cd-czx9t\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-dd85c69cd-czx9t (namespace clusters-service) on cluster ci01-j3332864-svc has averaged 238.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-17T00:03:50Z\n  Ended:       2026-09-17T00:13:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3332864-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-558d75db97-9xbgb\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container aro-hcp-backend in pod aro-hcp-backend-558d75db97-9xbgb (namespace aro-hcp) on cluster ci01-j3332864-svc has averaged 416.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-17T00:03:50Z\n  Ended:       2026-09-17T00:13:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3332864-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-72f4l\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-72f4l (namespace arobit) on cluster ci01-j3332864-svc has averaged 179.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T00:28:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3332864-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-dd85c69cd-vfcn8\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-dd85c69cd-vfcn8 (namespace clusters-service) on cluster ci01-j3332864-svc has averaged 230.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100373409937494016","occurred_at":"2026-09-16T23:56:58Z","signature_id":"1e3921ef26cf400f3e3eaae679c2c7938f1ecf3228c5ec7fae0e28ef8fa1159f","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100354655547559936","occurred_at":"2026-09-16T22:42:23Z","signature_id":"940a5892f0bde8defb7629a002baedf9228b8f92a2cf58838e303561286c491d","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100353767223332864","occurred_at":"2026-09-16T22:38:55Z","signature_id":"e71be1b08c965527fccf228721850a63e0d7ab82463979308010bd46b7d5afcc","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351996258160640","occurred_at":"2026-09-16T22:31:53Z","signature_id":"fc7a5f2beeb914b7f9d5e98e516ffb3d0e395b2c10277b6a8a41faeeb997a062","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351264033345536","occurred_at":"2026-09-16T22:22:06Z","signature_id":"38cba13ec304a2d65c8ec3a2bb0373cae43840dfeeed1f7a55cc2d8c840f0488","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351223038218240","occurred_at":"2026-09-16T22:21:29Z","signature_id":"fbe220275de66dc82c17bd70379b1968bfca47f26279aaeb942365f7083c7aae","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100346022097063936","occurred_at":"2026-09-16T22:08:08Z","signature_id":"5098cde8f56024f9b94cf40f2a466897158f3f3994b796f3a58cc2d52fcb7504","pr_number":4878,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100344321164185600","occurred_at":"2026-09-16T22:01:23Z","signature_id":"f7080c270925219a311e60803887fccb7900c43032060412377f7a288ab7c269","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100343556559343616","occurred_at":"2026-09-16T21:58:20Z","signature_id":"180e6e60fd74213dba13f27436fee7ff492e29f8f76edd5be5c54b377947d936","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100335879556960256","occurred_at":"2026-09-16T21:27:50Z","signature_id":"71b22a11302a00be21922c71aa067e721f76475093f648235ac53a36e12bcb39","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100335557920952320","occurred_at":"2026-09-16T21:26:33Z","signature_id":"ceaa91f4efdcdb0d1c542d52b327737fcf8ae4ccbf48f0c17aab82acbcc99238","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100331448274784256","occurred_at":"2026-09-16T21:10:14Z","signature_id":"30e6e0d7b43f0480d6e10d35a84534dd048930cbdf424724d15cc0834916efb3","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100328454825185280","occurred_at":"2026-09-16T20:58:20Z","signature_id":"cfb1506b480d229252a933ef91e34400b8d02a4638f13cabb01dc1d4175a17b8","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100327783220645888","occurred_at":"2026-09-16T20:55:40Z","signature_id":"18e89a273ad3f5766b2dd6e52e34589ba49632a0e92e5d7b81de104647e14a35","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"29b55dd4d895c72d0d59b9865df950f870325198fd7e112f1858c956d6c3812a","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"4ef72abc8b9948fd9bead06d17d6310bcbc699657b6ee721fd16b3106ac79918","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100314659356676096","occurred_at":"2026-09-16T20:03:31Z","signature_id":"d4858edb57d6296d8d8b3f66a13945cd04dcb772e32b66ac68791e2b7d64dc15","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"078c270d3b43c225b1670f600e86beb8fac31540b45fed0edb0e8f2b88023a38","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6904/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100310890917138432","occurred_at":"2026-09-16T19:48:32Z","signature_id":"ba7f166d742007eec805103f097b0619e99ae0ecb99c10d11604e7456c1148b3","pr_number":6904,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309676229922816","occurred_at":"2026-09-16T19:43:43Z","signature_id":"d7f945bb6e41f7832c5eef4f84c2ae943bce6eff034300bf30a51c4a754e1cdc","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309343839719424","occurred_at":"2026-09-16T19:42:24Z","signature_id":"efa1057a04dfc40b6e037e249cb0dc21b2c09a49fd06c904c78795387f5897e8","pr_number":6403,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309325879709696","occurred_at":"2026-09-16T19:42:19Z","signature_id":"c35a6da3cb62c984e490fa20921e5205ae8f9b5d5f473a81f2bc4d92f07256ce","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309243423887360","occurred_at":"2026-09-16T19:42:00Z","signature_id":"4e635a5a55fd9c70e99f4fc90a819fcbe842ed4f80ffca35e85d712a75adda03","pr_number":4878,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100299874200195072","occurred_at":"2026-09-16T19:04:46Z","signature_id":"94b33034edb32ac378faab020acb7aa1cb586b97112ae6c5f8f64b5e0f39f1b3","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298749917007872","occurred_at":"2026-09-16T19:00:16Z","signature_id":"569b1e0aed53209c579e2ada482c82175262b769185ccf3a5bafd789625f23b0","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298527354654720","occurred_at":"2026-09-16T18:59:25Z","signature_id":"2515546ca4a2ed9cc66209f7f5a861f329da8544f35e1712e6bef48626b727f0","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298495381475328","occurred_at":"2026-09-16T18:59:17Z","signature_id":"b7bbff7c4bc8e38db57964016b3e5dd22cbf69711555869ce39fed0d057ef5fa","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100294017282478080","occurred_at":"2026-09-16T18:41:29Z","signature_id":"d6b769c42dfc7af863ab56e421478bc2130ed7b8030470653f093468c2ae9445","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100293442402783232","occurred_at":"2026-09-16T18:39:12Z","signature_id":"6d07e5dea3d8417306b6aad1d07607ec094c290a529075bc5ae0f6c507ae8dc7","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100285588732121088","occurred_at":"2026-09-16T18:08:00Z","signature_id":"3165e7829bfab81efada837f8a5a846a713826a1a775685756db96ad2a2e7a0b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100280971763912704","occurred_at":"2026-09-16T17:49:39Z","signature_id":"5241ddfc74d49c5ecad2616742ded97b4a5f9bf5fbeee0f248252a9d33303902","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100273649465430016","occurred_at":"2026-09-16T17:20:33Z","signature_id":"2aaecc7a8aa85a870eddb45d3a454574d39a17a2e1971a571ef303ebfa1b5dfa","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","signature_id":"4aa09e305fda5dad99f7aa547324f1cc9a5313a1ed785f8832369357abebfa2a","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","signature_id":"e80538656acc5e08b78c0da3cbda2f5f642ff686a707fd278f435ee95c791d0f","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100262131709513728","occurred_at":"2026-09-16T16:34:47Z","signature_id":"8bca23390b6e128f806583b6f43999dfe3e0891c3867f9b00f777f7b70551bad","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100260298874163200","occurred_at":"2026-09-16T16:27:30Z","signature_id":"641a1726bac71eedb2360eec9a405c0d1e826983dcd4ea335ba7c7dfa3cebe9d","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100260058020450304","occurred_at":"2026-09-16T16:26:33Z","signature_id":"9a105f0d73b76a92d17a2de5a3542ed76121642b109339640b150bdbd8718601","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100253677548736512","occurred_at":"2026-09-16T16:01:02Z","signature_id":"e26ae0379e9c2f78baac16193a817ee1d4bd6be52f5b22f001aad146994821f2","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100253157446651904","occurred_at":"2026-09-16T15:59:08Z","signature_id":"bb91f870f2d6651db04eb26377a6ab0f560e56353470294354ffb9ea8c0407f9","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252538870697984","occurred_at":"2026-09-16T15:56:40Z","signature_id":"984ec4e1000694cd92d63a32c0f7a6e5d7b7f677e37cfce55ad670237d8b5da0","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252533153861632","occurred_at":"2026-09-16T15:56:39Z","signature_id":"59e4de1f8e513f5cc08bd4e593ae29483ec56ee54e8e92faff2ba25b8713cc09","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"fde97b6bc2d22cd4f0aa0ec9116922422f6896f61d1b4b08310d88dfdf610931","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"5176305f0eeec8bbbaa6ff316c0143a9416bf749550533697343b8a801ca28cd","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6923/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100244030364323840","occurred_at":"2026-09-16T15:22:38Z","signature_id":"3af7515b2be64cbe2958e1955e72cb60cdb6b11e0681c1a634bbc8ed9f8e15d6","pr_number":6923,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100243112008880128","occurred_at":"2026-09-16T15:19:04Z","signature_id":"8cecf1b875c80d5aaad598063b3c383cfb7ef9d6225c392907d6632715156b7b","pr_number":6923,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6726/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100185438990897152","occurred_at":"2026-09-16T11:30:02Z","signature_id":"5d5dbfb417b6897cd24a2f1e6ad614604212fa121b9f81e032502d4c16e94ed3","pr_number":6726,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":30,"runs_affected":30,"failed_runs":30,"run_impact_percent":3.9946737683089215,"anchor_occurrences":30,"after_last_push_count":10,"trend_counts":[0,0,0,0,0,0,0,30],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":30,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":30}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T23:46:37Z\n  Ended:       2026-09-17T00:05:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7559936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssdiuonvecl8bulup4napa18plli3hl-v5g6h1u7r3y6b9q\", pod=\"router-77b5d8dffb-95c5m\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2ssdiuonvecl8bulup4napa18plli3hl-v5g6h1u7r3y6b9q/router-77b5d8dffb-95c5m has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-16T23:46:37Z\n  Ended:       2026-09-17T00:05:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7559936-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssdisk0mggfhm9130ika04epdikqlho-s5n1l0v6o6a7b8a\", pod=\"router-55df5c6c57-b4j8r\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2ssdisk0mggfhm9130ika04epdikqlho-s5n1l0v6o6a7b8a/router-55df5c6c57-b4j8r has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T23:38:16Z\n  Ended:       2026-09-17T00:05:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8160640-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssdff0k383bfd7grp5qnbnt5v4r1b0u-d3b2r2a4t0o4l6a\", pod=\"router-5564cd696f-nb8mv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2ssdff0k383bfd7grp5qnbnt5v4r1b0u-d3b2r2a4t0o4l6a/router-5564cd696f-nb8mv has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T23:31:05Z\n  Ended:       2026-09-16T23:50:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3345536-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssdc08lq86mikr3m7qtfj8ulr4bs703-h7y5g4s0a4a5j6c\", pod=\"router-5bb8b7c8bd-thvsf\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2ssdc08lq86mikr3m7qtfj8ulr4bs703-h7y5g4s0a4a5j6c/router-5bb8b7c8bd-thvsf has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100354655547559936","occurred_at":"2026-09-16T22:42:23Z","signature_id":"b66349955715ed443f1362f9a218926c741bbba15a7053b8231a572f7bce0c04","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351996258160640","occurred_at":"2026-09-16T22:31:53Z","signature_id":"e2e746b818d8319297b0cfc1ce06bb87bc7386a6e269943bf13e78c9d13237ab","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100351264033345536","occurred_at":"2026-09-16T22:22:06Z","signature_id":"db6b8a56aaf730fdd696fc547f30dac94889302e62174f2dd2508fcd332898b0","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100344321164185600","occurred_at":"2026-09-16T22:01:23Z","signature_id":"600a7515642ad65bbfc931d4dd2a57d1a3c95b9594a86121e772644509ea9e97","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100335879556960256","occurred_at":"2026-09-16T21:27:50Z","signature_id":"c599c77a6372b3affcde70e85dd51dd5124ab9de3cabd625edc84ba8a133bbfb","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100327783220645888","occurred_at":"2026-09-16T20:55:40Z","signature_id":"4c70e8e6dfdf51fe895bb78044f498f38b34d7b31ddb973e4ed72258e72ee568","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"d21baacf92b823fbe47d168aff240c5f90998a22314dd5cc365610b36ceba21b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"ee56a79fd88c61821c015922fd0060df48fa8ea6f29e86e41766eb43042fab32","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"d8457d1570ea90a26bcd98437f37670575aae28bffe0f986f9b004d2d9264040","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298749917007872","occurred_at":"2026-09-16T19:00:16Z","signature_id":"4fe55ebf2e3687f10b470f9b50ef984b7f154b6ecd6235b377b7abf35f5fe025","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298495381475328","occurred_at":"2026-09-16T18:59:17Z","signature_id":"ed3bf953568238e27f1b875c5f5f12b7c39936c7b777f8037f83dc0b1f8c2feb","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","signature_id":"e4b9c749f68fadead7aeb206b02a1f85e9ae2c2c7142b04d727f3089de1f8541","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100262131709513728","occurred_at":"2026-09-16T16:34:47Z","signature_id":"b7c6ba500fe6358c309a0b082682885bdc0c4016fb309f71ff96ef107142bcca","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"8a55cdb17b2e0960799afe76d087110a5ecf992561f62b085684e8010de63d22","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"d167cd058619d1445e1dc6c6ed93bc2e291c8bbbd9eba4b08df3c14423f77517","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"4778f344e9b25530d8b9156641a99f6b2838c230f7532738e3f69e081b9f7a7e","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"045a81b731fe6b760898ced74dc07c1acf462fd8638ae785f456d496a795842e","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100213017273700352","occurred_at":"2026-09-16T13:19:37Z","signature_id":"422af2d2e106cf074701d11ee92eac11656adc2f791fd92e942ac51d5ff857ef","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","signature_id":"aede2952cb21cd0c07ff4ac502b4109b4236290ff6f43397958c99820cf77924","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","signature_id":"3a6cb6146cbf8c095d871f7807f59ec635628a6cc72ba515bf71641eb6ad5bd1","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"7451ba3db591daddff6044c850b50a806f031b44b26c0a9cb44f8b82e28dd244","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189503086923776","occurred_at":"2026-09-16T11:46:06Z","signature_id":"382d6e1a50a58344a999c679c7ef69f1f3c041161e556f0be9d14b621d84b09e","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100176649516486656","occurred_at":"2026-09-16T10:54:59Z","signature_id":"f201db6f99c06ca51a3a0a3934aae0846a6ea6ba393747b6f6edc6872fb99a72","pr_number":6952,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161384225443840","occurred_at":"2026-09-16T09:54:27Z","signature_id":"ae8593e2f93d516941e79280e7827e07a691be6004452673f6399cae8b22f618","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100153419804184576","occurred_at":"2026-09-16T09:22:48Z","signature_id":"3a248c85aeccd44a74b5e82dc19ae5115320308561bbe219e225440971c85f86","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"cf50089acb0846a23f15800cf9d1588e5a42034409429bcfeb7f8b4bdce19e86","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"74727d2f774b690907a06926c56a326fbc39e84388995b5a3c7eceec1a64f678","pr_number":6879,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6962/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132023019507712","occurred_at":"2026-09-16T07:57:47Z","signature_id":"22a7d0041fe9eca9fd2523ee31fb7da254fed5da06967c558019791f8f2e970a","pr_number":6962,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"05b02974c8e5aa95fe3326e72aadd1d9c24fabbae417684be2c2a88ea3d78009","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100124639266082816","occurred_at":"2026-09-16T07:28:27Z","signature_id":"1bb0a319d54a133be2e86c848056c08984c613c97a376359e9e2876ee27aa821","pr_number":6856,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":22,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.0652463382157125,"anchor_occurrences":22,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,22],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20260901preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001400078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-dqgbxbhrnvkj\" to finish deleting: GET https://rp.j6676096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38120612-3f62-4d83-b1c9-f450e308488b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38120612-3f62-4d83-b1c9-f450e308488b\",\n      \"name\": \"38120612-3f62-4d83-b1c9-f450e308488b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:57:03.974907545Z\",\n      \"endTime\": \"2026-09-16T21:21:04.12131988Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssb46njkre58n4p9mol1i33anuaqjtv still exists (deletion dispatched at 2026-09-16T20:57:05Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001400078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-dqgbxbhrnvkj\" to finish deleting: GET https://rp.j6676096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38120612-3f62-4d83-b1c9-f450e308488b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38120612-3f62-4d83-b1c9-f450e308488b\",\n      \"name\": \"38120612-3f62-4d83-b1c9-f450e308488b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:57:03.974907545Z\",\n      \"endTime\": \"2026-09-16T21:21:04.12131988Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssb46njkre58n4p9mol1i33anuaqjtv still exists (deletion dispatched at 2026-09-16T20:57:05Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0008e6240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-x4ht6zckdvtk\" to finish deleting: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/675795c0-1d48-4443-bf17-96b820fb9dab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/675795c0-1d48-4443-bf17-96b820fb9dab\",\n      \"name\": \"675795c0-1d48-4443-bf17-96b820fb9dab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:56:31.8857491Z\",\n      \"endTime\": \"2026-09-16T21:20:37.108128013Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssari2k27ic1890mj0nf8obfposs9em still exists (deletion dispatched at 2026-09-16T20:56:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0008e6240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-x4ht6zckdvtk\" to finish deleting: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/675795c0-1d48-4443-bf17-96b820fb9dab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/675795c0-1d48-4443-bf17-96b820fb9dab\",\n      \"name\": \"675795c0-1d48-4443-bf17-96b820fb9dab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:56:31.8857491Z\",\n      \"endTime\": \"2026-09-16T21:20:37.108128013Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssari2k27ic1890mj0nf8obfposs9em still exists (deletion dispatched at 2026-09-16T20:56:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001910318\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tfkgg85wrhjp\" to finish deleting: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/40b6c313-a567-4259-bb1a-298290750f51\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/40b6c313-a567-4259-bb1a-298290750f51\",\n      \"name\": \"40b6c313-a567-4259-bb1a-298290750f51\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:55:14.718059601Z\",\n      \"endTime\": \"2026-09-16T21:19:17.098694741Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssarn8fr2snie5k70mjfefs0cg1miti still exists (deletion dispatched at 2026-09-16T20:55:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001910318\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tfkgg85wrhjp\" to finish deleting: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/40b6c313-a567-4259-bb1a-298290750f51\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/40b6c313-a567-4259-bb1a-298290750f51\",\n      \"name\": \"40b6c313-a567-4259-bb1a-298290750f51\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:55:14.718059601Z\",\n      \"endTime\": \"2026-09-16T21:19:17.098694741Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssarn8fr2snie5k70mjfefs0cg1miti still exists (deletion dispatched at 2026-09-16T20:55:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100314659356676096","occurred_at":"2026-09-16T20:03:31Z","signature_id":"06e7cdde30cfbab4250217a65b741f57948c596ead351300b63debe1170a60a3","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"62f925cd8f107a148e561a08817704dd4da0dc08360a885c312c5776ffe6894d","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"e259668ae1165ff12f8afd3ff5be66f96028094b8d861c7cbdff5a5c0c31f019","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"460210ebabcf024c04114b174b6545f46b58b1c28d70303d86e3b1dce093c204","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"a552e8119646d9c743aa9a252024bde026b39e6cefa5828c2a635099d2355f2a","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"c81fbcddae76e3d46ce74bd7eeec6dbb65a70aa3f34997838af6880b8bc4b471","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"f38603c674107979a7e5113b087deeb85d932689f0a71354d02c3879c176e60d","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"09330ba162b6af089c34735c70eca9ddd91f6516659badb8a358d7c06fa7a1f1","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"4baa98b743ff58863c2c3f11ae6c7794d41c41f07100fe61766b0978f6746f19","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"6b6d7951973760ce40ada8345eab0fc3affb480bfb3af182b9b92a268d16b16b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"cf0c7c2f7324da55d7b02b3480eb3ca7ec6d10efdd79ac18d6876f5a4e4fc8b8","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"32832ce99401d97a958b7dcdfe20140118a267411f3e842c201974409fd354fb","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"47db8067bcef5f852d6be174031d8a4891925f676b5063754561044e04b33b85","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"6422616296d3a5a4112b796133cccebf1e701aef5be2522f1440d40d7f73f94f","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"e114928abafbdcce9fc1b5a0f9590defe570e1bf5ceed7138ffdb8bb94282914","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"e94f3f5e69e8e136647e4790e270a1391748aa6703a074a6fa38b2687beb9db4","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"ea4b0a4eb2c285656f848886af6bb6a27818c286a7b34ef24973fc4e223cf7a2","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222891458039808","occurred_at":"2026-09-16T13:58:52Z","signature_id":"1cb320d43390d047edfe367eca720f9ae752f792b2067b9a7d668b21e611b437","pr_number":6952,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"16a76a4ed4fe90d47f48a9657d0216e8d1f87acf39a8dad4d8ba5c30878fe100","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100188464501755904","occurred_at":"2026-09-16T11:42:04Z","signature_id":"139ea4eb9a07c176246f76b251911c3691a6cbbed5076e752b52b4042c281baf","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100188464501755904","occurred_at":"2026-09-16T11:42:04Z","signature_id":"85767451e4aa0503e59ddd278738b04f06b742245943e9225dbc7e3560e04b05","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100188464501755904","occurred_at":"2026-09-16T11:42:04Z","signature_id":"d4cf5fc8868b958b193d13f740ed514c1bae0dd9b8489c2b78271801c3f259d7","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":10,"runs_affected":10,"failed_runs":10,"run_impact_percent":1.3315579227696406,"anchor_occurrences":10,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,0,10],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":10,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":4},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":6}],"full_error_samples":["time=2026-09-16T21:39:23.882Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-16T21:39:24.651Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-16T21:39:25.419Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=1957e733d563fe8d01011d2966a166813e9314307467a323647f090022c9d7a7 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1475712-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F1957e733d563fe8d01011d2966a166813e9314307467a323647f090022c9d7a7\ntime=2026-09-16T22:09:23.556Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-09-16T17:32:16.355Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-16T17:32:17.069Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-16T17:32:18.139Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=2220b5a771141f6ca7f14ed4462ae3b3d70e20f12f11d2768e401b8114aa657f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7836672-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2220b5a771141f6ca7f14ed4462ae3b3d70e20f12f11d2768e401b8114aa657f\ntime=2026-09-16T18:02:16.065Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-09-16T14:44:36.842Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-16T14:44:37.777Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-16T14:44:38.583Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=00e0f6eaf1a87ba23e25957419627b2f2bf902c90c40b4a9a0680ec1b94026a9 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1035008-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F00e0f6eaf1a87ba23e25957419627b2f2bf902c90c40b4a9a0680ec1b94026a9\ntime=2026-09-16T15:14:36.624Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100332359231475712","occurred_at":"2026-09-16T21:13:51Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100270038647836672","occurred_at":"2026-09-16T17:06:12Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100228886901035008","occurred_at":"2026-09-16T14:22:39Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6779/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100198040831791104","occurred_at":"2026-09-16T12:20:07Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6779,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194885893099520","occurred_at":"2026-09-16T12:07:35Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194111955931136","occurred_at":"2026-09-16T12:04:19Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100192526756483072","occurred_at":"2026-09-16T11:58:12Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100190163022909440","occurred_at":"2026-09-16T11:48:49Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6694/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100169554075324416","occurred_at":"2026-09-16T10:26:55Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6694,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100119215724826624","occurred_at":"2026-09-16T07:06:53Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6972,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.0652463382157125,"anchor_occurrences":8,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,0,8],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":8}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T17:39:57Z\n  Ended:       2026-09-16T17:45:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0450304-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0450304-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T17:27:30Z\n  Ended:       2026-09-16T17:34:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3861632-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3861632-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T16:39:46Z\n  Ended:       2026-09-16T17:12:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6091136-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6091136-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100260058020450304","occurred_at":"2026-09-16T16:26:33Z","signature_id":"84adc99394ffdde58d8777dabe660ecf4888bc0533bab685e83dea23a8c7fc1a","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252533153861632","occurred_at":"2026-09-16T15:56:39Z","signature_id":"eb143e51cc4e677a9811a011abde7f9846dfb86171fd5089044d86a86bc3c30e","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"4891bae57c16737882a8247821ec7f6d4e17417de53801a0bcb72589d84d269e","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100228663373991936","occurred_at":"2026-09-16T14:21:48Z","signature_id":"af0482283bab09054590813e51480bea798a5743b7bcc33fa9c63b58d8645bbe","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"6113c0f8fe30ddb0cec1586532a7684c6dc0ec614d444f96448deb0f4ad3a170","pr_number":6879,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132038643290112","occurred_at":"2026-09-16T07:57:51Z","signature_id":"bf9a145d432cea73856e5e5cd0fb2d2856d4a87e3d9fdb2343a2f93ed338bb52","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6923/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100110113200148480","occurred_at":"2026-09-16T06:30:43Z","signature_id":"fac8088074452240e37582cff853c6a9cc2e040277fbf34b005056861933699a","pr_number":6923,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100104517457547264","occurred_at":"2026-09-16T06:08:29Z","signature_id":"5f54bcb248196b637bd98df597f0841d329c7e41ad40465254245b7458522f8c","pr_number":6894,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":0.9320905459387483,"anchor_occurrences":7,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,0,7],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011165a0\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-2nsvjcl7j6m6\" to finish creating: GET https://rp.j3803776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a27b380d-13cc-4abe-9932-546cf6483df0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a27b380d-13cc-4abe-9932-546cf6483df0\",\n      \"name\": \"a27b380d-13cc-4abe-9932-546cf6483df0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T21:08:44.335887928Z\",\n      \"endTime\": \"2026-09-16T21:37:44.000335858Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011165a0\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-2nsvjcl7j6m6\" to finish creating: GET https://rp.j3803776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a27b380d-13cc-4abe-9932-546cf6483df0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a27b380d-13cc-4abe-9932-546cf6483df0\",\n      \"name\": \"a27b380d-13cc-4abe-9932-546cf6483df0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T21:08:44.335887928Z\",\n      \"endTime\": \"2026-09-16T21:37:44.000335858Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000db98e0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-5kbxx5fn4nnf\" to finish creating: GET https://rp.j2783232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff402ee6-ae9a-437a-b814-5ad8ed11cf52\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff402ee6-ae9a-437a-b814-5ad8ed11cf52\",\n      \"name\": \"ff402ee6-ae9a-437a-b814-5ad8ed11cf52\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T19:26:24.241613754Z\",\n      \"endTime\": \"2026-09-16T19:55:27.054305259Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000db98e0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-5kbxx5fn4nnf\" to finish creating: GET https://rp.j2783232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff402ee6-ae9a-437a-b814-5ad8ed11cf52\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff402ee6-ae9a-437a-b814-5ad8ed11cf52\",\n      \"name\": \"ff402ee6-ae9a-437a-b814-5ad8ed11cf52\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T19:26:24.241613754Z\",\n      \"endTime\": \"2026-09-16T19:55:27.054305259Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-vlzgbk with version 4.21.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006bee60\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-vlzgbk: failed waiting for cluster=\"np-version-upgrade-cluster-vlzgbk\" in resourcegroup=\"rg-np-version-upgrade-vlzgbk-6gtpz2tlsds2\" to finish creating: GET https://rp.j2846464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83de4d68-f488-460e-8446-69209da0d691\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83de4d68-f488-460e-8446-69209da0d691\",\n      \"name\": \"83de4d68-f488-460e-8446-69209da0d691\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:51:23.682356624Z\",\n      \"endTime\": \"2026-09-16T18:20:27.523942323Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-vlzgbk with version 4.21.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006bee60\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-vlzgbk: failed waiting for cluster=\"np-version-upgrade-cluster-vlzgbk\" in resourcegroup=\"rg-np-version-upgrade-vlzgbk-6gtpz2tlsds2\" to finish creating: GET https://rp.j2846464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83de4d68-f488-460e-8446-69209da0d691\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83de4d68-f488-460e-8446-69209da0d691\",\n      \"name\": \"83de4d68-f488-460e-8446-69209da0d691\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:51:23.682356624Z\",\n      \"endTime\": \"2026-09-16T18:20:27.523942323Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"7037bf26441097d6041fad63a8ad90e3d7ac2c0bc29160794aafceabed76fa0d","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100293442402783232","occurred_at":"2026-09-16T18:39:12Z","signature_id":"3d2d19c49b473c9b2b5c1bd828e9238e2a0bdd4c4f8d89fe2be19a6ea937a4b7","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","signature_id":"4c86e6598dc5b904362415f87179a0f883f34b4b44e009896f6a7f0de46090e2","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"9ae40ec334e2759457083c312bb3ac6d6fef401546cf0c8751fc4d293a56e796","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","signature_id":"f792cb486e1be9a347881acf7c3d2cd96a120fd226d905b4a6c6fa830ffd406b","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6726/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100185438990897152","occurred_at":"2026-09-16T11:30:02Z","signature_id":"681376edcfb91afa0eef4e45ea02cce5668cf8a6c8cbfde456dd6c90a347a910","pr_number":6726,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132115432607744","occurred_at":"2026-09-16T07:58:09Z","signature_id":"76ed85811ec3a7dce6c999043c0cc9d5ea6c96b382b975abd6e4372f70524c64","pr_number":6964,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":7,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":7,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,7],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000861120\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-pkbzl9cwqzmf\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7bc0c172-a0d6-40dc-821f-14c51cede823\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7bc0c172-a0d6-40dc-821f-14c51cede823\",\n      \"name\": \"7bc0c172-a0d6-40dc-821f-14c51cede823\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:07:59.260634412Z\",\n      \"endTime\": \"2026-09-16T15:37:00.334559736Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000861120\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-pkbzl9cwqzmf\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7bc0c172-a0d6-40dc-821f-14c51cede823\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7bc0c172-a0d6-40dc-821f-14c51cede823\",\n      \"name\": \"7bc0c172-a0d6-40dc-821f-14c51cede823\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:07:59.260634412Z\",\n      \"endTime\": \"2026-09-16T15:37:00.334559736Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:206]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002a80740\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-czltd5xxrlvj\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/180c48f3-76f8-4d48-bcf4-3b4e29dffab7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/180c48f3-76f8-4d48-bcf4-3b4e29dffab7\",\n      \"name\": \"180c48f3-76f8-4d48-bcf4-3b4e29dffab7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:10:24.910079282Z\",\n      \"endTime\": \"2026-09-16T15:39:30.319726782Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:206]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002a80740\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-czltd5xxrlvj\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/180c48f3-76f8-4d48-bcf4-3b4e29dffab7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/180c48f3-76f8-4d48-bcf4-3b4e29dffab7\",\n      \"name\": \"180c48f3-76f8-4d48-bcf4-3b4e29dffab7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:10:24.910079282Z\",\n      \"endTime\": \"2026-09-16T15:39:30.319726782Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:93]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e96840\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-222ntj7gxqx2\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/774c9323-3626-4141-aa8c-a5f0d7e57e9c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/774c9323-3626-4141-aa8c-a5f0d7e57e9c\",\n      \"name\": \"774c9323-3626-4141-aa8c-a5f0d7e57e9c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:07:38.983252228Z\",\n      \"endTime\": \"2026-09-16T15:36:40.328092646Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:93]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e96840\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-222ntj7gxqx2\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/774c9323-3626-4141-aa8c-a5f0d7e57e9c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/774c9323-3626-4141-aa8c-a5f0d7e57e9c\",\n      \"name\": \"774c9323-3626-4141-aa8c-a5f0d7e57e9c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:07:38.983252228Z\",\n      \"endTime\": \"2026-09-16T15:36:40.328092646Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"1081dc1f7ba277c79eab9706dd413878fd827376a9850ff6a2e23d1b663c5155","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"51fb4ad7b246c62ce6ea07724802f78706075364625cc9b0dc59f38a98c159cf","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"52a3e55e5accf233dc0c4dcc31ad7c4bd1abab3ca6c7016627f681951931597f","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"8eb85a382d8ce34de42657ea34ddc180803991e921b8e70810a099fe550ece42","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"b323b711e4f15f03edb6542a780e728151255e7713e3d463bc33d5461bb8ccbf","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"daf2b2ff378776852d48a0372497ce4c540ea79fca95c306e900dac48324c4cd","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"ea6f45f7f8317d27652ca2ba1a9ea637325a720fb6e451e052632539521bf95b","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.7989347536617842,"anchor_occurrences":6,"after_last_push_count":2,"also_in":["INT","PROD","STG"],"trend_counts":[0,0,0,0,0,0,0,6],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bb5820\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-xxwwr4bjs848\" to finish creating: GET https://rp.j8426240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13d2af80-0cd5-4fee-b2fe-b55a2b2248e3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13d2af80-0cd5-4fee-b2fe-b55a2b2248e3\",\n      \"name\": \"13d2af80-0cd5-4fee-b2fe-b55a2b2248e3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T18:00:53.783577799Z\",\n      \"endTime\": \"2026-09-16T18:19:53.25990245Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bb5820\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-xxwwr4bjs848\" to finish creating: GET https://rp.j8426240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13d2af80-0cd5-4fee-b2fe-b55a2b2248e3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13d2af80-0cd5-4fee-b2fe-b55a2b2248e3\",\n      \"name\": \"13d2af80-0cd5-4fee-b2fe-b55a2b2248e3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T18:00:53.783577799Z\",\n      \"endTime\": \"2026-09-16T18:19:53.25990245Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008cd0a0\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-jvwfr7\" in resourcegroup=\"rg-np-skip-minor-jvwfr7-7qvvgr9wwfbq\" to finish creating: GET https://rp.j2424320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c6ba019-61f3-4ad5-922c-02cdd95f824e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c6ba019-61f3-4ad5-922c-02cdd95f824e\",\n      \"name\": \"6c6ba019-61f3-4ad5-922c-02cdd95f824e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T14:13:13.201859753Z\",\n      \"endTime\": \"2026-09-16T14:32:13.373200335Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008cd0a0\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-jvwfr7\" in resourcegroup=\"rg-np-skip-minor-jvwfr7-7qvvgr9wwfbq\" to finish creating: GET https://rp.j2424320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c6ba019-61f3-4ad5-922c-02cdd95f824e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c6ba019-61f3-4ad5-922c-02cdd95f824e\",\n      \"name\": \"6c6ba019-61f3-4ad5-922c-02cdd95f824e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T14:13:13.201859753Z\",\n      \"endTime\": \"2026-09-16T14:32:13.373200335Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001ff26c0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-g5s8ftp9hh9t\" to finish creating: GET https://rp.j4241024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06ef0feb-5e7e-4993-bd90-39657ca016bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06ef0feb-5e7e-4993-bd90-39657ca016bf\",\n      \"name\": \"06ef0feb-5e7e-4993-bd90-39657ca016bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T12:18:53.259035342Z\",\n      \"endTime\": \"2026-09-16T12:37:53.675934016Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001ff26c0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-g5s8ftp9hh9t\" to finish creating: GET https://rp.j4241024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06ef0feb-5e7e-4993-bd90-39657ca016bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06ef0feb-5e7e-4993-bd90-39657ca016bf\",\n      \"name\": \"06ef0feb-5e7e-4993-bd90-39657ca016bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T12:18:53.259035342Z\",\n      \"endTime\": \"2026-09-16T12:37:53.675934016Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","signature_id":"11b5fac45f89b3b06fa90df1c77a750f6663c0d038abc00b0bfa209370a40fa8","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100212360202424320","occurred_at":"2026-09-16T13:16:57Z","signature_id":"fca0d524c5ac315fe836fb88fd7ec89c3926bdf9a49356a4f7143c3576597659","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100177539514241024","occurred_at":"2026-09-16T10:58:39Z","signature_id":"4035d63aa7b2b3c2922026477a9808247a9246b7c7d1f67b92c709fe54aaae23","pr_number":6952,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100170313131102208","occurred_at":"2026-09-16T10:29:56Z","signature_id":"f02c430f8d6e71033486f073da10a12fce79eb5455b645129c51f20c2adc1b42","pr_number":6886,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100164075236364288","occurred_at":"2026-09-16T10:05:09Z","signature_id":"05b8123730c99aae90bf4a2eb92268ce4dedb8de1e1191eb555f549c617a26dd","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"b85f0369f91fe32b9c9c2062c2413bb6bf553dd1bd69f6e51a7a27c7ba432aee","pr_number":6791,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.7989347536617842,"anchor_occurrences":6,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,0,6],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":6}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T21:44:54Z\n  Ended:       2026-09-16T21:51:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3803776-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.109:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3803776/providers/microsoft.monitor/accounts/hcps-j3803776\", namespace=\"ocm-arohcpci01-2ssben4sp2jl4ug8rvisqeb9vbtmfdbc-n8l2c7a7y0n1i2r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-gndsj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2ssben4sp2jl4ug8rvisqeb9vbtmfdbc-n8l2c7a7y0n1i2r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-16T21:45:52Z\n  Ended:       2026-09-16T22:01:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3803776-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.109:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3803776/providers/microsoft.monitor/accounts/hcps-j3803776\", namespace=\"ocm-arohcpci01-2ssben4sp2jl4ug8rvisqeb9vbtmfdbc-n8l2c7a7y0n1i2r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-gndsj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2ssben4sp2jl4ug8rvisqeb9vbtmfdbc-n8l2c7a7y0n1i2r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T17:11:51Z\n  Ended:       2026-09-16T17:29:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/hcps-j6091136\", namespace=\"ocm-arohcpci01-2ss7f44euk2o4nqkrp28r72hngjs9t5m-g7f9d6m4t8e6w1m\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2ss7f44euk2o4nqkrp28r72hngjs9t5m-g7f9d6m4t8e6w1m/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T16:30:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7769984-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7769984/providers/microsoft.monitor/accounts/hcps-j7769984\", namespace=\"ocm-arohcpci01-2ss6631b7kcrqufnkep1rdniihr6hk1s-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-n4qlf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2ss6631b7kcrqufnkep1rdniihr6hk1s-cilium-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T16:30:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7769984-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7769984/providers/microsoft.monitor/accounts/hcps-j7769984\", namespace=\"ocm-arohcpci01-2ss6631b7kcrqufnkep1rdniihr6hk1s-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-n4qlf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2ss6631b7kcrqufnkep1rdniihr6hk1s-cilium-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"0d965c3262ee69273c199442b4bed217de7dd1b47c2e6f6c022895186517f6a2","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"314e25f4f6bda2d5748f9fe5aa89f88eaeb725849f7741eef5e0355c2394136c","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"d5ed3091019088eb341f9671128472e74176d896eca057722a5636069c20079f","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","signature_id":"18b82d4d472bbc8192b3a0be8a58b9abe4cf0da273e711ee46164983b7c2c490","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"278a7d810c61d983530553cb6ff8366e5a4d79b77f9c15651d9ebcb3dbeb9e4e","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"89db9db5751f9f51402e00b217747d3b3ea1c62487d699b8eeaa41599ef274e2","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"52bfcbbc720001a7392b6a1e08f396579b61443952ed16c2c80c98fe6498c690","failure_pattern":"alert [svc] userJourneyFrontendLatencyP996h30m fired","search_query":"alert [svc] userJourneyFrontendLatencyP996h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP996h30m does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.7989347536617842,"anchor_occurrences":6,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,0,6],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP996h30m does not fire","occurrences":6}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T09:29:21Z\n  Ended:       2026-09-16T09:49:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatencyP996h30m\", cluster=\"ci01-j5155200-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5155200/providers/microsoft.monitor/accounts/services-j5155200\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n  Description: Frontend p99 latency on cluster ci01-j5155200-svc is above 1s on both the 6h and 30m windows (p99 \u003c 1s ARM sync SLO miss).","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T09:46:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatencyP996h30m\", cluster=\"ci01-j3290112-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3290112/providers/microsoft.monitor/accounts/services-j3290112\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n  Description: Frontend p99 latency on cluster ci01-j3290112-svc is above 1s on both the 6h and 30m windows (p99 \u003c 1s ARM sync SLO miss).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T08:53:50Z\n  Ended:       2026-09-16T09:06:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatencyP996h30m\", cluster=\"ci01-j9843328-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9843328/providers/microsoft.monitor/accounts/services-j9843328\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n  Description: Frontend p99 latency on cluster ci01-j9843328-svc is above 1s on both the 6h and 30m windows (p99 \u003c 1s ARM sync SLO miss)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132075855155200","occurred_at":"2026-09-16T07:57:59Z","signature_id":"4e60a01b4827c68ff57b07edc9eab558ce75aad5112478ce2737bee6696f871b","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132038643290112","occurred_at":"2026-09-16T07:57:51Z","signature_id":"9daba40c60f37faf2af38abc3e3474d5917ef206c95bff1353f0bdaf3df9ac3b","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"cbb7931aab2d0ed1916b8d84363dfe1ffc05cf35d9ce851f977710db9cdf561f","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6971/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100127403039789056","occurred_at":"2026-09-16T07:39:25Z","signature_id":"02815353fa0a66ddf20b88ae30c92eac6b08249d84094735fa13cf0c3952861f","pr_number":6971,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6923/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100110113200148480","occurred_at":"2026-09-16T06:30:43Z","signature_id":"d87f770fb784724db359656072889a1f3015ce39f593db987908535455d1ee42","pr_number":6923,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100104517457547264","occurred_at":"2026-09-16T06:08:29Z","signature_id":"d21b60b6e4bcd8f18d5041389daab6c5b24d976173fd091a6cd2e7e5a098f762","pr_number":6894,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":6,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.5326231691078562,"anchor_occurrences":6,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,6],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:90]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eaac80\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-5sd5p2zrd22p\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb287309-affe-4cb6-ba60-0ab97101fffb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb287309-affe-4cb6-ba60-0ab97101fffb\",\n      \"name\": \"cb287309-affe-4cb6-ba60-0ab97101fffb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T21:22:15.559693502Z\",\n      \"endTime\": \"2026-09-16T21:51:23.369287505Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:90]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eaac80\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-5sd5p2zrd22p\" to finish creating: GET https://rp.j3686400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb287309-affe-4cb6-ba60-0ab97101fffb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb287309-affe-4cb6-ba60-0ab97101fffb\",\n      \"name\": \"cb287309-affe-4cb6-ba60-0ab97101fffb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T21:22:15.559693502Z\",\n      \"endTime\": \"2026-09-16T21:51:23.369287505Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:584]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101ca40\u003e: \n    failed to create HCP cluster np-downgrade-ptnt6v: failed waiting for cluster=\"np-downgrade-ptnt6v\" in resourcegroup=\"rg-np-downgrade-ptnt6v-s6586bctkvvs\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b81923cc-2583-43fb-a0ea-baa197fb0d6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b81923cc-2583-43fb-a0ea-baa197fb0d6f\",\n      \"name\": \"b81923cc-2583-43fb-a0ea-baa197fb0d6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:07:45.747660708Z\",\n      \"endTime\": \"2026-09-16T15:36:50.286764481Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:584]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101ca40\u003e: \n    failed to create HCP cluster np-downgrade-ptnt6v: failed waiting for cluster=\"np-downgrade-ptnt6v\" in resourcegroup=\"rg-np-downgrade-ptnt6v-s6586bctkvvs\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b81923cc-2583-43fb-a0ea-baa197fb0d6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b81923cc-2583-43fb-a0ea-baa197fb0d6f\",\n      \"name\": \"b81923cc-2583-43fb-a0ea-baa197fb0d6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:07:45.747660708Z\",\n      \"endTime\": \"2026-09-16T15:36:50.286764481Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-vm24lv8gf4qb/cluster-candidate-4-21-llvd77 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011b0380\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-llvd77: failed waiting for cluster=\"cluster-candidate-4-21-llvd77\" in resourcegroup=\"rg-candidate-4-21-vm24lv8gf4qb\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a4429abc-10d8-4151-ad1a-6f415aa5475f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a4429abc-10d8-4151-ad1a-6f415aa5475f\",\n      \"name\": \"a4429abc-10d8-4151-ad1a-6f415aa5475f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T10:52:02.803955113Z\",\n      \"endTime\": \"2026-09-16T11:21:09.695382854Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-vm24lv8gf4qb/cluster-candidate-4-21-llvd77 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011b0380\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-llvd77: failed waiting for cluster=\"cluster-candidate-4-21-llvd77\" in resourcegroup=\"rg-candidate-4-21-vm24lv8gf4qb\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a4429abc-10d8-4151-ad1a-6f415aa5475f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a4429abc-10d8-4151-ad1a-6f415aa5475f\",\n      \"name\": \"a4429abc-10d8-4151-ad1a-6f415aa5475f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T10:52:02.803955113Z\",\n      \"endTime\": \"2026-09-16T11:21:09.695382854Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"2ffe72f7718510ccd07fc00e17f58bc0013af474b7348754fe62e7d3c4440904","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"f53d0a32dc007711392e3ac9a6346e4050a79f13a86cf1b09a36647de1c455fb","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"55ad0310438d8f56a4a299f6bab3b8eead74cd3a5f7c47eb326a8464bccf50a4","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"91a400cbca3134720c48834459e3002d3cce5a22f2d380c716d75f65f9d75177","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"b82de203b62ff6756c30d031383d9f2c1cda2b52933916b1090a4b903f7e89b9","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"25f5f479c873e8672b346f86e1757b6d6cdd4c81dcdc1af82909656949ba3709","pr_number":6879,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"40f7a558a9f14e3ad84a0dc62150fb9e1dcec198cbfa1c00cbbb104cbc63f3a7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":6,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":6,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,6],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0028a4438\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-j5zkl24x6bwt\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecb9f85-876c-467d-91c9-980ce11c9827\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecb9f85-876c-467d-91c9-980ce11c9827\",\n      \"name\": \"5ecb9f85-876c-467d-91c9-980ce11c9827\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:49:19.070350042Z\",\n      \"endTime\": \"2026-09-16T10:13:22.487477443Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0028a4438\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-j5zkl24x6bwt\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecb9f85-876c-467d-91c9-980ce11c9827\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ecb9f85-876c-467d-91c9-980ce11c9827\",\n      \"name\": \"5ecb9f85-876c-467d-91c9-980ce11c9827\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:49:19.070350042Z\",\n      \"endTime\": \"2026-09-16T10:13:22.487477443Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00153e0f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-9dwsgxpprw8c\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\",\n      \"name\": \"9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:44:48.885828252Z\",\n      \"endTime\": \"2026-09-16T10:08:52.4506315Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00153e0f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-9dwsgxpprw8c\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\",\n      \"name\": \"9e39e5c6-0d40-4bf6-9634-0b867d3cf17c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:44:48.885828252Z\",\n      \"endTime\": \"2026-09-16T10:08:52.4506315Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000012f78\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-g9bvp2\" in resourcegroup=\"rg-candidate-4-20-pmdbfsk6v6zd\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/834c2939-b42e-48bd-b340-960d8826a429\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/834c2939-b42e-48bd-b340-960d8826a429\",\n      \"name\": \"834c2939-b42e-48bd-b340-960d8826a429\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:49:50.424611242Z\",\n      \"endTime\": \"2026-09-16T10:13:52.492147666Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000012f78\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-g9bvp2\" in resourcegroup=\"rg-candidate-4-20-pmdbfsk6v6zd\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/834c2939-b42e-48bd-b340-960d8826a429\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/834c2939-b42e-48bd-b340-960d8826a429\",\n      \"name\": \"834c2939-b42e-48bd-b340-960d8826a429\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:49:50.424611242Z\",\n      \"endTime\": \"2026-09-16T10:13:52.492147666Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"18c69429807d38cfdd701e251877ce552b6dddaac55af8cb470eb1279a95b526","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"232d76b4c8604eadbe09c019eea03e4b35e8f44b8a88fbcca5afcc985e967ce4","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"9410511803cbf1e3c159d5decef593399c3f8d02958271b7b1f469d232f45e6e","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"9635d65ddbc8d8183553acfbe1c7e3ebf7cfb49f71fb8721193391b355cf2fbd","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"ba6d9f57396d35aba73dd64e7c229f8d7cbb408e58e239c7c2b2065df4e28656","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"d539c69fd361c2a857c348ec75ef524c1710267ecef6f193ceb123bc3e8bce18","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.6657789613848203,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,5],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:738]: failed to create nodepool npdg-422-420\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a2c540\u003e: \n    failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-86c2sh\" in resourcegroup=\"rg-np-dg-minor-86c2sh-42qzgk6zdhqx\" to finish creating: GET https://rp.j4654720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b168b436-e53c-4032-86e5-822b4fbf8df0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b168b436-e53c-4032-86e5-822b4fbf8df0\",\n      \"name\": \"b168b436-e53c-4032-86e5-822b4fbf8df0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T19:54:19.817293417Z\",\n      \"endTime\": \"2026-09-16T19:54:53.786912449Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:738]: failed to create nodepool npdg-422-420\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a2c540\u003e: \n    failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-86c2sh\" in resourcegroup=\"rg-np-dg-minor-86c2sh-42qzgk6zdhqx\" to finish creating: GET https://rp.j4654720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b168b436-e53c-4032-86e5-822b4fbf8df0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b168b436-e53c-4032-86e5-822b4fbf8df0\",\n      \"name\": \"b168b436-e53c-4032-86e5-822b4fbf8df0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T19:54:19.817293417Z\",\n      \"endTime\": \"2026-09-16T19:54:53.786912449Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002420258\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-hgxdxgb7d5pf\" to finish creating: GET https://rp.j3912704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0b2672-7030-4135-9a4f-3ac73ff9d19d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0b2672-7030-4135-9a4f-3ac73ff9d19d\",\n      \"name\": \"bc0b2672-7030-4135-9a4f-3ac73ff9d19d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T18:52:16.312207682Z\",\n      \"endTime\": \"2026-09-16T18:52:49.366400193Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002420258\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-hgxdxgb7d5pf\" to finish creating: GET https://rp.j3912704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0b2672-7030-4135-9a4f-3ac73ff9d19d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0b2672-7030-4135-9a4f-3ac73ff9d19d\",\n      \"name\": \"bc0b2672-7030-4135-9a4f-3ac73ff9d19d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T18:52:16.312207682Z\",\n      \"endTime\": \"2026-09-16T18:52:49.366400193Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-96ddmv\" for cluster \"cluster-candidate-5-0-96ddmv\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d846a0\u003e: \n    failed to create NodePool np-96ddmv: failed waiting for nodepool=\"np-96ddmv\" for cluster \"cluster-candidate-5-0-96ddmv\" in resourcegroup=\"rg-candidate-5-0-dxgjw5725ck7\" to finish creating: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75dfa262-dc67-4818-8d1e-eacd838b47fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75dfa262-dc67-4818-8d1e-eacd838b47fd\",\n      \"name\": \"75dfa262-dc67-4818-8d1e-eacd838b47fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:23:26.413052673Z\",\n      \"endTime\": \"2026-09-16T15:24:04.373688035Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-96ddmv\" for cluster \"cluster-candidate-5-0-96ddmv\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d846a0\u003e: \n    failed to create NodePool np-96ddmv: failed waiting for nodepool=\"np-96ddmv\" for cluster \"cluster-candidate-5-0-96ddmv\" in resourcegroup=\"rg-candidate-5-0-dxgjw5725ck7\" to finish creating: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75dfa262-dc67-4818-8d1e-eacd838b47fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75dfa262-dc67-4818-8d1e-eacd838b47fd\",\n      \"name\": \"75dfa262-dc67-4818-8d1e-eacd838b47fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:23:26.413052673Z\",\n      \"endTime\": \"2026-09-16T15:24:04.373688035Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298527354654720","occurred_at":"2026-09-16T18:59:25Z","signature_id":"bf7ebbd181ec7aa5127a5794e1778bd8570a9bdf75c05ec0454a8c04349e2cae","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100280971763912704","occurred_at":"2026-09-16T17:49:39Z","signature_id":"e4e6c14d747d24bff47997187743a4306c16de10ca6e0660e678007cc3c9c76a","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"37839b6f366af376a2e2dee46f1b956ca9ee9897a768408f797bc256ab2e722b","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"080a9e45d6f4d800d1c8aca157df99a9c02fa085354fe52205b1fcea2ffcae50","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132038643290112","occurred_at":"2026-09-16T07:57:51Z","signature_id":"fb548ce2c81d7ac7aefb5c63ff2cf1623addd485c6b89a7815a4071a2a0cf5c7","pr_number":6503,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.6657789613848203,"anchor_occurrences":5,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,0,5],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":5}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T18:21:59Z\n  Ended:       2026-09-16T18:42:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2846464-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2846464/providers/microsoft.monitor/accounts/hcps-j2846464\", namespace=\"ocm-arohcpci01-2ss8i74sks91slsm7oliign0ld9qcsch-q0f6x6b6a5y9c2t\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fbvzq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2ss8i74sks91slsm7oliign0ld9qcsch-q0f6x6b6a5y9c2t/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-16T18:21:59Z\n  Ended:       2026-09-16T18:29:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2846464-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2846464/providers/microsoft.monitor/accounts/hcps-j2846464\", namespace=\"ocm-arohcpci01-2ss8i74sks91slsm7oliign0ld9qcsch-q0f6x6b6a5y9c2t\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fbvzq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2ss8i74sks91slsm7oliign0ld9qcsch-q0f6x6b6a5y9c2t/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T16:56:51Z\n  Ended:       2026-09-16T17:24:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/hcps-j6091136\", namespace=\"ocm-arohcpci01-2ss7f44euk2o4nqkrp28r72hngjs9t5m-g7f9d6m4t8e6w1m\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2ss7f44euk2o4nqkrp28r72hngjs9t5m-g7f9d6m4t8e6w1m/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T14:02:53Z\n  Ended:       2026-09-16T14:27:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9619840-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.80:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9619840/providers/microsoft.monitor/accounts/hcps-j9619840\", namespace=\"ocm-arohcpci01-2ss4t4fdubst8s5b8tltfi3bmb1s95er-u3f5u3d4c4j0m6r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-q22d9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2ss4t4fdubst8s5b8tltfi3bmb1s95er-u3f5u3d4c4j0m6r/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-16T14:02:53Z\n  Ended:       2026-09-16T14:22:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9619840-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.80:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9619840/providers/microsoft.monitor/accounts/hcps-j9619840\", namespace=\"ocm-arohcpci01-2ss4t4fdubst8s5b8tltfi3bmb1s95er-u3f5u3d4c4j0m6r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-q22d9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2ss4t4fdubst8s5b8tltfi3bmb1s95er-u3f5u3d4c4j0m6r/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","signature_id":"f812f1c6853ffe5ec0bbae33597f7ac343264a944ebc861a726251e8dce290ce","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"b995781a6685881ffdb2f7b53822eec544ca3973eb89f51c67b3781f51c35c18","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","signature_id":"67413ecc4fbc527d42baea24debbddc611e8d90812ae968414dd499232adf3e3","pr_number":6898,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6726/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100185438990897152","occurred_at":"2026-09-16T11:30:02Z","signature_id":"e361abead0603e0446eac1b70560a55fbc49aa8e7044098f3920029a67178a2a","pr_number":6726,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132115432607744","occurred_at":"2026-09-16T07:58:09Z","signature_id":"2202a1a6bca53c40fa9b99a6151f7edd9b592782635ef9dfab434b81cf067278","pr_number":6964,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.6657789613848203,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,5],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":5}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T22:06:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.104:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1475712/providers/microsoft.monitor/accounts/services-j1475712\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-59mfc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T22:06:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.104:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1475712/providers/microsoft.monitor/accounts/services-j1475712\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-59mfc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:55:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7836672/providers/microsoft.monitor/accounts/services-j7836672\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-zdqrc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:55:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7836672/providers/microsoft.monitor/accounts/services-j7836672\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5979b8454d-zdqrc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T16:45:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T16:45:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T16:46:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T16:46:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100332359231475712","occurred_at":"2026-09-16T21:13:51Z","signature_id":"617d8bea63f7c0a8e5aa65e252453013f2f87eaabc747dea5d193edf0ea92b6a","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100270038647836672","occurred_at":"2026-09-16T17:06:12Z","signature_id":"659a105790bd191ed63ba5bb7e543dad17b2528556030da7e70281aa38365890","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"8cefc20a001466adc6a61bb43c8407f0170786f35f4348a8d83cb4fc19e10669","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161854826352640","occurred_at":"2026-09-16T09:56:19Z","signature_id":"73664eccd132c9d84475a61b44857f75314a6635b79c98d2ae19c6ecf756089a","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132066686406656","occurred_at":"2026-09-16T07:57:56Z","signature_id":"358902433200cc0a70d0c371110e19da053abecdbf25175a02fa1df21c363226","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.6657789613848203,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,5],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":5}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T21:55:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-dgkdj\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-dgkdj has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T21:55:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-cb282\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-cb282 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T21:55:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-hdnbp\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-hdnbp has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T21:55:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1475712-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-f58cs\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-f58cs has been in a non-ready state for longer than 5 minutes.","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:45:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-59s8j\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-59s8j has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:45:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-9zlkb\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-9zlkb has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:45:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-t8hdl\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-t8hdl has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:45:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7836672-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-49tpv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-49tpv has been in a non-ready state for longer than 5 minutes.","alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T16:36:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-cwxxr\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod kube-system/aks-secrets-store-csi-driver-cwxxr has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-16T16:36:14Z\n  Ended:       2026-09-16T17:19:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-28181016-vmss000000-vxff5\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod kube-system/eraser-aks-system-28181016-vmss000000-vxff5 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T16:36:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-xfmjp\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod kube-system/ama-metrics-node-xfmjp has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100332359231475712","occurred_at":"2026-09-16T21:13:51Z","signature_id":"4cfee31556334d3f7789166186ab36ba11bfce4b40f49a38dd4a3b6152068d51","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100270038647836672","occurred_at":"2026-09-16T17:06:12Z","signature_id":"ce202b7ce3d72dcf4b71c51db3c021d84810e4ab274c64dae9e86007cff0fec7","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"59897c5f85f7527efbbab27b1d42a8c63b62d2873c4723537141e6e4cb09a74c","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161854826352640","occurred_at":"2026-09-16T09:56:19Z","signature_id":"f46fa77bbf961b7174b5b28e58ea0b344fb9d8b591b31027fca64620b0c955e9","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132066686406656","occurred_at":"2026-09-16T07:57:56Z","signature_id":"5c93d0c045993799007f8590467470292210fd23c8f5530b0425e5382165f354","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.5326231691078562,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,4],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":4}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T22:05:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j1475712-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j1475712-svc has been unreachable for 15 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:55:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j7836672-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j7836672-svc has been unreachable for 15 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T10:51:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j6352640-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j6352640-svc has been unreachable for 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100332359231475712","occurred_at":"2026-09-16T21:13:51Z","signature_id":"9b1a4c0162aa96d1ba2cfc1a58fe8450105df2f95a84e56b0c0206685fdb86fb","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100270038647836672","occurred_at":"2026-09-16T17:06:12Z","signature_id":"c1988f523c9b1951a937f22e869077937abcdb055a057f4fd9deb538ebdeeab8","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161854826352640","occurred_at":"2026-09-16T09:56:19Z","signature_id":"40d8f345b61cc5a9851d9600ef3889f30540178c4979cc679d48f241649fe011","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132066686406656","occurred_at":"2026-09-16T07:57:56Z","signature_id":"cdf1b6098bdd9c6cf95b2228b3d492b2e30cb0b5e195c0384f719de68055f224","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.5326231691078562,"anchor_occurrences":4,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,0,4],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":4}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T21:38:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6757888-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-bcnth\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-bcnth (namespace arobit) on cluster ci01-j6757888-svc has averaged 127.2% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T21:58:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9922816-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8kkxq\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-8kkxq (namespace arobit) on cluster ci01-j9922816-svc has averaged 125.6% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T19:40:11Z\n  Ended:       2026-09-16T19:46:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j2121088-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-rxj44\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-rxj44 (namespace arobit) on cluster ci01-j2121088-svc has averaged 126.6% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"7cedda9870b8f7ce3e9e32ac805564cf0bac96565ca3328c26875a1d70efde15","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309676229922816","occurred_at":"2026-09-16T19:43:43Z","signature_id":"3a277ae7c258e0424f569baffe0a35ab11aec77aa3384cc607f2298340f9545f","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100285588732121088","occurred_at":"2026-09-16T18:08:00Z","signature_id":"040c54783eeb571a32188be480f0e0617f1911469f8d08b133b147d373df8efb","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100273649465430016","occurred_at":"2026-09-16T17:20:33Z","signature_id":"de52e1f3238a7b10dc025230a0bc30a7b49fe207de3da3e23620f54404272671","pr_number":6894,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.5326231691078562,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,4],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"b9b50758a1aa006036c7c1ee58af90cd509c7b1900722ef0229b46ed2f5e8563","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100196783199096832","occurred_at":"2026-09-16T12:15:07Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132080049459200","occurred_at":"2026-09-16T07:58:01Z","signature_id":"817dbbd2f4e987cf783899d57ba9ab565857a252a91d7741c25fbfc506aff5e8","pr_number":6879,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.3994673768308921,"anchor_occurrences":4,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,4],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a9c270\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-fqmsfk\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-lqr7gtcm4srq\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\",\n      \"name\": \"3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:13:26.770029811Z\",\n      \"endTime\": \"2026-09-16T17:37:32.768384818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j3fnpr9m9cqfjaohaiq73k6sjvpi still exists (deletion dispatched at 2026-09-16T17:13:27Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a9c270\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-fqmsfk\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-lqr7gtcm4srq\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\",\n      \"name\": \"3f5ff4f5-7eef-4834-9b6c-c2e1109ad259\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:13:26.770029811Z\",\n      \"endTime\": \"2026-09-16T17:37:32.768384818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j3fnpr9m9cqfjaohaiq73k6sjvpi still exists (deletion dispatched at 2026-09-16T17:13:27Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014240f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xbmtbzvlpclx\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d85e799c-786e-4172-95d1-86693923282f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d85e799c-786e-4172-95d1-86693923282f\",\n      \"name\": \"d85e799c-786e-4172-95d1-86693923282f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:32:57.751739191Z\",\n      \"endTime\": \"2026-09-16T17:57:03.956555792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j51n0bvbqmvanc2bru5bjg1kuddr still exists (deletion dispatched at 2026-09-16T17:32:58Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014240f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xbmtbzvlpclx\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d85e799c-786e-4172-95d1-86693923282f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d85e799c-786e-4172-95d1-86693923282f\",\n      \"name\": \"d85e799c-786e-4172-95d1-86693923282f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:32:57.751739191Z\",\n      \"endTime\": \"2026-09-16T17:57:03.956555792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7j51n0bvbqmvanc2bru5bjg1kuddr still exists (deletion dispatched at 2026-09-16T17:32:58Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000012e40\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-wbh9jqvv44p7\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9db0308e-c5ac-4693-b367-d76dc77af863\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9db0308e-c5ac-4693-b367-d76dc77af863\",\n      \"name\": \"9db0308e-c5ac-4693-b367-d76dc77af863\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:29:44.443897591Z\",\n      \"endTime\": \"2026-09-16T15:53:44.471117038Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss64i7i766p1sol3r0dalpub5p7l53h still exists (deletion dispatched at 2026-09-16T15:29:46Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000012e40\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-wbh9jqvv44p7\" to finish deleting: GET https://rp.j8680064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9db0308e-c5ac-4693-b367-d76dc77af863\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9db0308e-c5ac-4693-b367-d76dc77af863\",\n      \"name\": \"9db0308e-c5ac-4693-b367-d76dc77af863\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:29:44.443897591Z\",\n      \"endTime\": \"2026-09-16T15:53:44.471117038Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss64i7i766p1sol3r0dalpub5p7l53h still exists (deletion dispatched at 2026-09-16T15:29:46Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"4b15da3ce1591e33d2696652650de2d0cdc6c841165c33a2a923881126078487","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"4f20aa230f0afb2f03a0196f22a9def6c79b679a7bc9bc79347900fd06c241a8","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"b79349f0a2b72f52e04da9ac4e8ef7179dd24862a80796349270662d77641269","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222891458039808","occurred_at":"2026-09-16T13:58:52Z","signature_id":"8491d54e1e6f9d99de4d3a50c4e2d20f3eb57611f24ff8bb3d09a72d4b3c8424","pr_number":6952,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"575b0de6701cb1cdcdd3b5c4e6523248471d3aa67c3c0c1fed243233e4014b33","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Monitor","search_query":"ERROR CODE: ResourceGroupNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"AMA Metrics service metrics from SVC and MGMT clusters should be present in Azure Monitor","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2663115845539281,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,4],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"AMA Metrics service metrics from SVC and MGMT clusters should be present in Azure Monitor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:74]: failed to look up HCP Prometheus endpoint\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00072e820\u003e: \n    failed to get workspace hcps-j3128064: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3128064/providers/Microsoft.Monitor/accounts/hcps-j3128064\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j3128064' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:74]: failed to look up HCP Prometheus endpoint\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00072e820\u003e: \n    failed to get workspace hcps-j3128064: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3128064/providers/Microsoft.Monitor/accounts/hcps-j3128064\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j3128064' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-j3128064 in resource group hcp-underlay-ci01-j3128064\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010489e0\u003e: \n    failed to get workspace services-j3128064: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3128064/providers/Microsoft.Monitor/accounts/services-j3128064\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j3128064' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-j3128064 in resource group hcp-underlay-ci01-j3128064\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010489e0\u003e: \n    failed to get workspace services-j3128064: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3128064/providers/Microsoft.Monitor/accounts/services-j3128064\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j3128064' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:74]: failed to look up HCP Prometheus endpoint\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c94e20\u003e: \n    failed to get workspace hcps-yt: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-yt/providers/Microsoft.Monitor/accounts/hcps-yt\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-yt' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:74]: failed to look up HCP Prometheus endpoint\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c94e20\u003e: \n    failed to get workspace hcps-yt: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-yt/providers/Microsoft.Monitor/accounts/hcps-yt\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-yt' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","signature_id":"1d95e250094820742e5d106c9a6198395eff4115c52cfbf3176662d0996cbb8f","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","signature_id":"2300bcf0e83155e35375beb6ab2cd2dd1e60df39d954749c7fd210ed41e13cde","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"4a840acfa7ba40488e61bba537bd235d758cbe0b71e71c1cdf5af1d8876188b2","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"e60b8a3f0f846e4caf38925534828b81ff9b59a90b098e189f25e9987eb8d737","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fe6b01560ac49646fd0a46c48f672cf2fc0ebbf844ea4576bb0cab7a46df7ce8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,4],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:103]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001f3c120\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sxmn5zhcsvjn\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a019abb2-ae16-4218-9707-6f7dc73a3eb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a019abb2-ae16-4218-9707-6f7dc73a3eb3\",\n      \"name\": \"a019abb2-ae16-4218-9707-6f7dc73a3eb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:08:36.527287031Z\",\n      \"endTime\": \"2026-09-16T15:37:40.20476418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:103]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001f3c120\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sxmn5zhcsvjn\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a019abb2-ae16-4218-9707-6f7dc73a3eb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a019abb2-ae16-4218-9707-6f7dc73a3eb3\",\n      \"name\": \"a019abb2-ae16-4218-9707-6f7dc73a3eb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:08:36.527287031Z\",\n      \"endTime\": \"2026-09-16T15:37:40.20476418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--7tj5b6r9d28k/cluster-hs-pre-rwztb2 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ee48a0\u003e: \n    failed to create HCP cluster cluster-hs-pre-rwztb2: failed waiting for cluster=\"cluster-hs-pre-rwztb2\" in resourcegroup=\"rg-hs-pre--7tj5b6r9d28k\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0015f221-0420-4231-b96b-76ce90032f9d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0015f221-0420-4231-b96b-76ce90032f9d\",\n      \"name\": \"0015f221-0420-4231-b96b-76ce90032f9d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:07:38.092292569Z\",\n      \"endTime\": \"2026-09-16T15:36:40.334174064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--7tj5b6r9d28k/cluster-hs-pre-rwztb2 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ee48a0\u003e: \n    failed to create HCP cluster cluster-hs-pre-rwztb2: failed waiting for cluster=\"cluster-hs-pre-rwztb2\" in resourcegroup=\"rg-hs-pre--7tj5b6r9d28k\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0015f221-0420-4231-b96b-76ce90032f9d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0015f221-0420-4231-b96b-76ce90032f9d\",\n      \"name\": \"0015f221-0420-4231-b96b-76ce90032f9d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:07:38.092292569Z\",\n      \"endTime\": \"2026-09-16T15:36:40.334174064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d8e640\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mldclqf9sr5h\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/704efcc4-4b42-4704-9ff8-f01712701a44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/704efcc4-4b42-4704-9ff8-f01712701a44\",\n      \"name\": \"704efcc4-4b42-4704-9ff8-f01712701a44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:09:00.206878787Z\",\n      \"endTime\": \"2026-09-16T15:38:00.28474108Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d8e640\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mldclqf9sr5h\" to finish creating: GET https://rp.j7769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/704efcc4-4b42-4704-9ff8-f01712701a44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/704efcc4-4b42-4704-9ff8-f01712701a44\",\n      \"name\": \"704efcc4-4b42-4704-9ff8-f01712701a44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:09:00.206878787Z\",\n      \"endTime\": \"2026-09-16T15:38:00.28474108Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"0bf39d4aab091ac51c471f05a7fa67a4d591d3eb93555a71c26170936f523c5f","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"a64a06ed18c7d9e35b003b4339972fe04157c007a8cd68d29875724bb6a184d4","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"b2b0b3c510ba3d2ec5bcda27a848edbf0f6e4130232ba677304593697d08e0b2","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"e41f400f85ccdcde13c1223820f617791e8453915fe2b4d9d6f910f58c9aac3e","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.3994673768308921,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":3}],"full_error_samples":["alert fired 5 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T17:00:59Z\n  Ended:       2026-09-16T17:19:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4807552-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-16T17:00:59Z\n  Ended:       2026-09-16T17:19:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4807552-svc\", component=\"backend\", name=\"datadump\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue datadump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-16T17:00:59Z\n  Ended:       2026-09-16T17:20:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4807552-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-16T17:02:01Z\n  Ended:       2026-09-16T17:19:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4807552-svc\", component=\"backend\", name=\"clusterclusterserviceupdatedispatch\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterclusterserviceupdatedispatch has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-16T17:09:02Z\n  Ended:       2026-09-16T17:19:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4807552-svc\", component=\"backend\", name=\"triggercontrolplaneupgrade\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue triggercontrolplaneupgrade has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T15:24:31Z\n  Ended:       2026-09-16T15:42:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8680064-svc\", component=\"backend\", name=\"dataplaneoidcfederation\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue dataplaneoidcfederation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T10:20:03Z\n  Ended:       2026-09-16T10:35:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4184576-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"660cafcaf8e83347ebad23ce568c8f5b0468c2f68c8566340ecac05676075f1d","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"fe76c2aef36843354f881968133b4c2405287f6a2f2ee3e1e7ab8ab4682897ce","pr_number":6983,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100153419804184576","occurred_at":"2026-09-16T09:22:48Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.3994673768308921,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.000s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0007c5690\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.000s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0007c5690\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.010s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001cb64b0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.010s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001cb64b0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0018f0700\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0018f0700\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100298495381475328","occurred_at":"2026-09-16T18:59:17Z","signature_id":"6b84028a0066d5af15d34aa2f3390d5b3b8c5439b097d46537c60eee00e895c1","pr_number":6977,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100243112008880128","occurred_at":"2026-09-16T15:19:04Z","signature_id":"e0666d1ee60ed5c6d4f663533ea9b4f979d5b8183e061f975c9361ca0cafa025","pr_number":6923,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100237146764873728","occurred_at":"2026-09-16T14:55:30Z","signature_id":"493948c9ce416bdc516799e228ff28ed99f8807ce0868ea1c0984eabe1af5cd2","pr_number":6752,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b414f524aec7c4ea3c6d464450b9af6f734740715a61aff1c3a8f53547f62641","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2663115845539281,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e4c880\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7vfjq4g5mpkz\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\",\n      \"name\": \"fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:25:02.43485078Z\",\n      \"endTime\": \"2026-09-16T15:54:04.537444594Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e4c880\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7vfjq4g5mpkz\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\",\n      \"name\": \"fa4772cb-eb8a-477d-b8e0-5d1f516c5a63\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:25:02.43485078Z\",\n      \"endTime\": \"2026-09-16T15:54:04.537444594Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d7dc40\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-7vbqmtrz9vrj\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a28fe40a-0fe9-4492-945f-7a7bb07d9075\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a28fe40a-0fe9-4492-945f-7a7bb07d9075\",\n      \"name\": \"a28fe40a-0fe9-4492-945f-7a7bb07d9075\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:24:46.641065736Z\",\n      \"endTime\": \"2026-09-16T15:53:54.422419253Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d7dc40\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-7vbqmtrz9vrj\" to finish creating: GET https://rp.j3034240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a28fe40a-0fe9-4492-945f-7a7bb07d9075\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a28fe40a-0fe9-4492-945f-7a7bb07d9075\",\n      \"name\": \"a28fe40a-0fe9-4492-945f-7a7bb07d9075\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:24:46.641065736Z\",\n      \"endTime\": \"2026-09-16T15:53:54.422419253Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009a0c20\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-xcwwm98r852j\" to finish creating: GET https://rp.j5673216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9e7e240-22cf-4186-a0b5-ed5964184230\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9e7e240-22cf-4186-a0b5-ed5964184230\",\n      \"name\": \"c9e7e240-22cf-4186-a0b5-ed5964184230\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T12:39:41.842014119Z\",\n      \"endTime\": \"2026-09-16T13:08:44.867300406Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009a0c20\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-xcwwm98r852j\" to finish creating: GET https://rp.j5673216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9e7e240-22cf-4186-a0b5-ed5964184230\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9e7e240-22cf-4186-a0b5-ed5964184230\",\n      \"name\": \"c9e7e240-22cf-4186-a0b5-ed5964184230\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T12:39:41.842014119Z\",\n      \"endTime\": \"2026-09-16T13:08:44.867300406Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"5f788a7a822406c498eab147df2fe3fad0c3f67d39005f710714ed9d67ac601d","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100231086113034240","occurred_at":"2026-09-16T14:31:25Z","signature_id":"976862ec83bbad292378adfc6b6876d3447587a76e0f813a8827a52bd7db4bbc","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100189557025673216","occurred_at":"2026-09-16T11:46:24Z","signature_id":"4c23de01cdd9b886d00d4c9a0959f33955c73d1a556a45314d353da1da388706","pr_number":6976,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001512108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-mwjj2q\" in resourcegroup=\"admin-credential-lifecycle-test-q4n7ckbgk7dt\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5124f997-7212-409c-99ae-fbd89a51e2f4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5124f997-7212-409c-99ae-fbd89a51e2f4\",\n      \"name\": \"5124f997-7212-409c-99ae-fbd89a51e2f4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:44:00.208113506Z\",\n      \"endTime\": \"2026-09-16T10:08:02.509863824Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss12vvsmbd0thdp0f88kc41u6pvi7c9 still exists (deletion dispatched at 2026-09-16T09:44:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001512108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-mwjj2q\" in resourcegroup=\"admin-credential-lifecycle-test-q4n7ckbgk7dt\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5124f997-7212-409c-99ae-fbd89a51e2f4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5124f997-7212-409c-99ae-fbd89a51e2f4\",\n      \"name\": \"5124f997-7212-409c-99ae-fbd89a51e2f4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:44:00.208113506Z\",\n      \"endTime\": \"2026-09-16T10:08:02.509863824Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss12vvsmbd0thdp0f88kc41u6pvi7c9 still exists (deletion dispatched at 2026-09-16T09:44:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0008820c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-hz87cjqncdvb\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e715185-33d5-494b-976c-25b2ab261cc3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e715185-33d5-494b-976c-25b2ab261cc3\",\n      \"name\": \"8e715185-33d5-494b-976c-25b2ab261cc3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:35:58.536453378Z\",\n      \"endTime\": \"2026-09-16T10:00:02.536562643Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss1371bjtto4nf5m6o0pid4pjvle6qc still exists (deletion dispatched at 2026-09-16T09:35:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0008820c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-hz87cjqncdvb\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e715185-33d5-494b-976c-25b2ab261cc3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8e715185-33d5-494b-976c-25b2ab261cc3\",\n      \"name\": \"8e715185-33d5-494b-976c-25b2ab261cc3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:35:58.536453378Z\",\n      \"endTime\": \"2026-09-16T10:00:02.536562643Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss1371bjtto4nf5m6o0pid4pjvle6qc still exists (deletion dispatched at 2026-09-16T09:35:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019ec138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-gql2drdp7tjr\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92c64afb-54ba-4f5e-a0ce-1a57e90c2b0b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92c64afb-54ba-4f5e-a0ce-1a57e90c2b0b\",\n      \"name\": \"92c64afb-54ba-4f5e-a0ce-1a57e90c2b0b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:34:43.828694597Z\",\n      \"endTime\": \"2026-09-16T09:58:52.519202639Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss1361u3o4ev8vfck5hk2a42dp30j3q still exists (deletion dispatched at 2026-09-16T09:34:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019ec138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-gql2drdp7tjr\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92c64afb-54ba-4f5e-a0ce-1a57e90c2b0b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/92c64afb-54ba-4f5e-a0ce-1a57e90c2b0b\",\n      \"name\": \"92c64afb-54ba-4f5e-a0ce-1a57e90c2b0b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:34:43.828694597Z\",\n      \"endTime\": \"2026-09-16T09:58:52.519202639Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss1361u3o4ev8vfck5hk2a42dp30j3q still exists (deletion dispatched at 2026-09-16T09:34:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"697bd9b6305361c1b5307d24f6145b0b596cf49460c3ec8e96a3eae19ea908a8","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"6ba2d7f149e18d77de06d3b79ce4ecb80b3fb20da43bf259cd0570583e9d7af6","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"ff4cffb5ab05b8596815bef0f9113f08c5ccdefbdae02ac3f027a27a7d4486ea","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f7e85048bd6764cc5537fb876507c8173ed1c8737be07b8c67fd9fcfdae58024","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001580078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dfbjv854d6w5\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\",\n      \"name\": \"d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:36:31.257109777Z\",\n      \"endTime\": \"2026-09-16T10:00:32.532474531Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss1375769r7k6ubu137jlbqtq9j35it still exists (deletion dispatched at 2026-09-16T09:36:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001580078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dfbjv854d6w5\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\",\n      \"name\": \"d99ec0f0-e1d1-41b0-b753-9ba0ef65c4b3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:36:31.257109777Z\",\n      \"endTime\": \"2026-09-16T10:00:32.532474531Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss1375769r7k6ubu137jlbqtq9j35it still exists (deletion dispatched at 2026-09-16T09:36:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011720a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-4cfzlz\" in resourcegroup=\"rg-zstream-upgrade-5-0-2bmmvg6nm4m6\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49e79c0-b564-4703-ad68-3e8fbb555ee7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49e79c0-b564-4703-ad68-3e8fbb555ee7\",\n      \"name\": \"a49e79c0-b564-4703-ad68-3e8fbb555ee7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:52:39.028633437Z\",\n      \"endTime\": \"2026-09-16T10:16:42.563449333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss136411ov65aeqqn1ut2t4iprrujv9 still exists (deletion dispatched at 2026-09-16T09:52:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011720a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-4cfzlz\" in resourcegroup=\"rg-zstream-upgrade-5-0-2bmmvg6nm4m6\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49e79c0-b564-4703-ad68-3e8fbb555ee7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a49e79c0-b564-4703-ad68-3e8fbb555ee7\",\n      \"name\": \"a49e79c0-b564-4703-ad68-3e8fbb555ee7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:52:39.028633437Z\",\n      \"endTime\": \"2026-09-16T10:16:42.563449333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss136411ov65aeqqn1ut2t4iprrujv9 still exists (deletion dispatched at 2026-09-16T09:52:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010be048\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-lq45ld6h888d\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76270801-127d-4225-97dc-06e57322bf5a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76270801-127d-4225-97dc-06e57322bf5a\",\n      \"name\": \"76270801-127d-4225-97dc-06e57322bf5a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:37:08.667361714Z\",\n      \"endTime\": \"2026-09-16T10:01:12.766789046Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss13j30utlk9cm4u8n21tm903korv38 still exists (deletion dispatched at 2026-09-16T09:37:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010be048\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-lq45ld6h888d\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76270801-127d-4225-97dc-06e57322bf5a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76270801-127d-4225-97dc-06e57322bf5a\",\n      \"name\": \"76270801-127d-4225-97dc-06e57322bf5a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:37:08.667361714Z\",\n      \"endTime\": \"2026-09-16T10:01:12.766789046Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss13j30utlk9cm4u8n21tm903korv38 still exists (deletion dispatched at 2026-09-16T09:37:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"3ce6a891cb0d07c7ce901cd0ee33f19037e602ebe5639fc8eb520176e22be57f","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"5a758cc0e5fd4d4f0a2773989c77d2a7c459fe910fa5486bed83da0d068c3f31","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"70756564bd668b5819cf49c7d07af12de78c4ca05f5acb0024a436a269a2a249","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2663115845539281,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:183]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:183]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100324480323686400","occurred_at":"2026-09-16T20:42:32Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410","failure_pattern":"ERROR CODE: DeploymentStackDeploymentFailed","search_query":"ERROR CODE: DeploymentStackDeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2663115845539281,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","occurrences":2}],"full_error_samples":["time=2026-09-16T18:22:02.494Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n  Kind: ARMStack\\n  Template: templates/monitoring.bicep\\n  Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-09-16T18:22:51.675Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/canadacentral/deploymentStackOperationStatus/1df7c411-d9ac-443d-ac5d-b95681f8419b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/canadacentral/deploymentStackOperationStatus/1df7c411-d9ac-443d-ac5d-b95681f8419b\\\",\\n  \\\"name\\\": \\\"1df7c411-d9ac-443d-ac5d-b95681f8419b\\\",\\n  \\\"status\\\": \\\"failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n    \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'd478c792-be07-45be-a743-97b4e94abf15'.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5982976/providers/Microsoft.Resources/deployments/nbyngfosfawbsx5vtxb2uo6lmfim2rbaopp3emm2zwlmm-260916186gla8\\\",\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n        \\\"details\\\": [\\n          {\\n            \\\"code\\\": \\\"BadRequest\\\",\\n            \\\"message\\\": \\\"Property name: Rules[0].Expression, Attempted value: null, Error: 1:237: parse error: unknown escape sequence U+002E '.' Activity ID: 95e8cbf4-5734-47ec-859e-d6ed81e0056f.\\\"\\n          }\\n        ]\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-16T17:10:05.170Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n  Kind: ARMStack\\n  Template: templates/monitoring.bicep\\n  Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-09-16T17:11:24.952Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/canadacentral/deploymentStackOperationStatus/aeca7e8d-50c1-47d0-a644-b291a00162ea\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/canadacentral/deploymentStackOperationStatus/aeca7e8d-50c1-47d0-a644-b291a00162ea\\\",\\n  \\\"name\\\": \\\"aeca7e8d-50c1-47d0-a644-b291a00162ea\\\",\\n  \\\"status\\\": \\\"failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n    \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'df67a523-14d6-42a2-9636-444dbb7d2763'.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2389376/providers/Microsoft.Resources/deployments/nbyngfosfawbsx5vtxb2uo6lmfim2rbaopp3emm2zwlmm-260916172yp3z\\\",\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n        \\\"details\\\": [\\n          {\\n            \\\"code\\\": \\\"BadRequest\\\",\\n            \\\"message\\\": \\\"Property name: Rules[0].Expression, Attempted value: null, Error: 1:237: parse error: unknown escape sequence U+002E '.' Activity ID: 91890f9b-76b2-4791-9c30-927a7253a007.\\\"\\n          }\\n        ]\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100283069305982976","occurred_at":"2026-09-16T17:57:59Z","signature_id":"2a5637582a78b2337c484f5a7a9ff7f9d84aa702c7b086e89182f1549eb13d57","pr_number":6965,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100265336732389376","occurred_at":"2026-09-16T16:47:31Z","signature_id":"2a5637582a78b2337c484f5a7a9ff7f9d84aa702c7b086e89182f1549eb13d57","pr_number":6965,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2663115845539281,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019540a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-kttjnq\" in resourcegroup=\"rg-np-downgrade-kttjnq-75tphq26mtkt\" to finish deleting: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/941782dd-efae-4093-940b-ca21585fc901\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/941782dd-efae-4093-940b-ca21585fc901\",\n      \"name\": \"941782dd-efae-4093-940b-ca21585fc901\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:46:21.836683685Z\",\n      \"endTime\": \"2026-09-16T21:10:27.160876667Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssarr47iva7e9en1e12mbat1fvbbvpq still exists (deletion dispatched at 2026-09-16T20:46:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019540a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-kttjnq\" in resourcegroup=\"rg-np-downgrade-kttjnq-75tphq26mtkt\" to finish deleting: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/941782dd-efae-4093-940b-ca21585fc901\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/941782dd-efae-4093-940b-ca21585fc901\",\n      \"name\": \"941782dd-efae-4093-940b-ca21585fc901\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:46:21.836683685Z\",\n      \"endTime\": \"2026-09-16T21:10:27.160876667Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssarr47iva7e9en1e12mbat1fvbbvpq still exists (deletion dispatched at 2026-09-16T20:46:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001654180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-vwpvr2srntsx\" to finish deleting: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9690275d-f0a2-4269-97f3-ff42287a50a3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9690275d-f0a2-4269-97f3-ff42287a50a3\",\n      \"name\": \"9690275d-f0a2-4269-97f3-ff42287a50a3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:08:51.225823952Z\",\n      \"endTime\": \"2026-09-16T15:32:51.218856846Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss5qjn2q2n7cj829v2pum0bhr8rsof2 still exists (deletion dispatched at 2026-09-16T15:08:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001654180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-vwpvr2srntsx\" to finish deleting: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9690275d-f0a2-4269-97f3-ff42287a50a3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9690275d-f0a2-4269-97f3-ff42287a50a3\",\n      \"name\": \"9690275d-f0a2-4269-97f3-ff42287a50a3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:08:51.225823952Z\",\n      \"endTime\": \"2026-09-16T15:32:51.218856846Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss5qjn2q2n7cj829v2pum0bhr8rsof2 still exists (deletion dispatched at 2026-09-16T15:08:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"9cfc23c00b3b6ff4e037fcfa63ecc4ff6f6077b374d4b670e533df57b7d90b23","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"46b2146b5bea1fbbe7547a18b5b7e0a96d274edde0ce207e2338267f45f60742","pr_number":6976,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2663115845539281,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":2}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T18:12:57Z\n  Ended:       2026-09-16T18:25:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2846464-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2846464/providers/microsoft.monitor/accounts/hcps-j2846464\", namespace=\"ocm-arohcpci01-2ss8i74sks91slsm7oliign0ld9qcsch-q0f6x6b6a5y9c2t\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6e854687-f7c2-439d-b129-bec4fba1965e\"\n  Description: Pod ocm-arohcpci01-2ss8i74sks91slsm7oliign0ld9qcsch-q0f6x6b6a5y9c2t/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-16T18:12:57Z\n  Ended:       2026-09-16T18:32:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2846464-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.181:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2846464/providers/microsoft.monitor/accounts/hcps-j2846464\", namespace=\"ocm-arohcpci01-2ss8i74sks91slsm7oliign0ld9qcsch-q0f6x6b6a5y9c2t\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6e854687-f7c2-439d-b129-bec4fba1965e\"\n  Description: Pod ocm-arohcpci01-2ss8i74sks91slsm7oliign0ld9qcsch-q0f6x6b6a5y9c2t/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T14:03:47Z\n  Ended:       2026-09-16T14:33:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9619840-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.80:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9619840/providers/microsoft.monitor/accounts/hcps-j9619840\", namespace=\"ocm-arohcpci01-2ss4t4fdubst8s5b8tltfi3bmb1s95er-u3f5u3d4c4j0m6r\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"aaf39014-4158-4621-b188-69047e816d57\"\n  Description: Pod ocm-arohcpci01-2ss4t4fdubst8s5b8tltfi3bmb1s95er-u3f5u3d4c4j0m6r/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","signature_id":"18090d26c3a2666e0dfd30f6dc97088df512304fef09dec58c535d88cec6f5bd","pr_number":6966,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100207889619619840","occurred_at":"2026-09-16T12:59:15Z","signature_id":"2c11c083dc96efa9f51e185909eda32654c395f47b2dbb78eaee76c2573a630d","pr_number":6898,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2663115845539281,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T17:11:15Z\n  Ended:       2026-09-16T17:17:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"klusterlet-2ss7f44euk2o4nqkrp28r72hngjs9t5m\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ss7f44euk2o4nqkrp28r72hngjs9t5m/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T15:50:51Z\n  Ended:       2026-09-16T15:58:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7769984-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.76:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7769984/providers/microsoft.monitor/accounts/services-j7769984\", namespace=\"klusterlet-2ss6631b7kcrqufnkep1rdniihr6hk1s\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-n4qlf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ss6631b7kcrqufnkep1rdniihr6hk1s/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"cd9f479af97dbe998875874266d9ea41c8e58990a5cf6ae60b1ca79fabc876a2","pr_number":6982,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"3b4c127aae1159c832d5991591f49782774b64887255f529b831eb0acc43aae1","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2663115845539281,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.000s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.000s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100268367708426240","occurred_at":"2026-09-16T16:59:34Z","signature_id":"ee111baab05f6b8ee26f353e865437e1bffda7140d9fc154398f141cd776e6f7","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100164075236364288","occurred_at":"2026-09-16T10:05:09Z","signature_id":"186ac175cc37e33f1eaa9e81000b9ad2fcf1670566cc6849f36ba3c782d224b2","pr_number":6688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2663115845539281,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.006s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00199ac80\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:35223-\u003e172.30.0.10:53: read: connection refused\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.006s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00199ac80\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:35223-\u003e172.30.0.10:53: read: connection refused\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.007s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0017cdba0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:34702-\u003e172.30.0.10:53: read: connection refused\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.007s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0017cdba0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:34702-\u003e172.30.0.10:53: read: connection refused\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100294017282478080","occurred_at":"2026-09-16T18:41:29Z","signature_id":"30b7eed5c08665fa2cef128c104b6bc116a828b17cff3a0328ac6b1a7aaf5b24","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","signature_id":"e12e1b463aa599590cba976eac3785b82a07ad6fd298aa2e1664ebd292e43daa","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c71ebc2a93d9fc10a75dcaf39a2b1325de6379c3891238b64d1e4e8f4281df9a","failure_pattern":"failed to get kusto cluster \u003ckusto-cluster\u003e from config: configuration: key kusto not found","search_query":"failed to get kusto cluster name from config: configuration: key kusto not found","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2663115845539281,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0012e60a0\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0012e60a0\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000a540c0\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000a540c0\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100194205623128064","occurred_at":"2026-09-16T12:04:52Z","signature_id":"9b7d539624448efd4996e6b535e22c740df41bfa52b50eab26733ce795540f15","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100129038159843328","occurred_at":"2026-09-16T07:45:55Z","signature_id":"12729704797975e796e396a1c7d5eb085f228e272a9762c5cf68851df1b74ece","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2663115845539281,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","occurrences":2}],"full_error_samples":["time=2026-09-16T10:35:02.273Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T10:35:02.716Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6352640-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6352640-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:ad132446-7529-417f-aff8-31d826ffb6f2 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#ad132446-7529-417f-aff8-31d826ffb6f2 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T10:35:02.716Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:35:02.863Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T10:35:03.042Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T10:35:03.042Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:35:03.143Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:45:05.949Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T10:45:05.949Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T10:46:05.955Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T10:46:06.637Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6352640-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6352640-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:ad132446-7529-417f-aff8-31d826ffb6f2 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#ad132446-7529-417f-aff8-31d826ffb6f2 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T10:46:06.637Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:07.013Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T10:46:07.408Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:07.408Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:07.670Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-16T10:46:08.488Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:08.489Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:08.888Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:09.401Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T10:46:09.747Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T10:46:09.959Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:10.361Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:10.729Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:46:11.065Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T10:46:11.065Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:46:11.148Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-16T10:56:12.973Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T10:56:12.973Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T10:57:12.977Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T10:57:13.486Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6352640-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6352640-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:ad132446-7529-417f-aff8-31d826ffb6f2 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#ad132446-7529-417f-aff8-31d826ffb6f2 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T10:57:13.486Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:13.807Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T10:57:14.174Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:14.174Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:14.404Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-16T10:57:15.329Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:15.329Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:15.688Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:15.987Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T10:57:16.345Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T10:57:16.535Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:16.747Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:16.984Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T10:57:17.208Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T10:57:17.208Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T10:57:17.300Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-16T11:05:02.358Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-16T11:05:02.359Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","time=2026-09-16T08:43:27.720Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T08:43:28.143Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6406656-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6406656-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:510fcd49-73a6-4643-b104-d1da046c7a87 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#510fcd49-73a6-4643-b104-d1da046c7a87 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T08:43:28.143Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:43:28.283Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T08:43:28.434Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T08:43:28.434Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:43:28.493Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:53:31.202Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T08:53:31.203Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T08:54:31.204Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T08:54:31.661Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6406656-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6406656-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:510fcd49-73a6-4643-b104-d1da046c7a87 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#510fcd49-73a6-4643-b104-d1da046c7a87 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T08:54:31.661Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:31.919Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T08:54:32.413Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:32.413Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:32.686Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-16T08:54:33.497Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:33.498Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:33.902Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:34.404Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T08:54:34.757Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T08:54:34.953Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:35.182Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:35.544Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T08:54:36.068Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T08:54:36.068Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T08:54:36.176Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-16T09:04:37.727Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T09:04:37.727Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-16T09:05:37.733Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-16T09:05:38.373Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j6406656-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:94cee54eb85d08891b179bd74be48632bf6e7968f046225abb6074b4655891d0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:100m memory:256Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j6406656-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:0af23e7645c07968b9bd20f5c4a2966b34ccdaf8a05812b5eb5a3a26c53f24b5 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:510fcd49-73a6-4643-b104-d1da046c7a87 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#510fcd49-73a6-4643-b104-d1da046c7a87 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-16T09:05:38.373Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:38.676Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-16T09:05:39.023Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:39.023Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:39.181Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-16T09:05:40.018Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:40.018Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:40.264Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:40.639Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T09:05:41.058Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-16T09:05:41.352Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:41.683Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:42.063Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-16T09:05:42.276Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-16T09:05:42.276Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-16T09:05:42.359Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-16T09:13:27.645Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-16T09:13:27.646Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161854826352640","occurred_at":"2026-09-16T09:56:19Z","signature_id":"64a318e95cdcfb12234888d476e960be3b284497df057c04c923477d134c5a2a","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132066686406656","occurred_at":"2026-09-16T07:57:56Z","signature_id":"d08b0cbd15edc9cffd195b1bd96feff56d9f329451cf7393af1293efbc3c0c96","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"50012cc84060fc59db5e840f9933bf1a542baf6598061d9d1e54da53aa955055","failure_pattern":"permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews","search_query":"permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2663115845539281,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.100s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001341f60\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.100s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001341f60\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.093s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000ff3040\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/volumeattachments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiextensions.k8s.io/customresourcedefinitions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list policy/poddisruptionbudgets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list autoscaling/horizontalpodautoscalers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list coordination.k8s.io/leases: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list discovery.k8s.io/endpointslices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list certificates.k8s.io/certificatesigningrequests: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list scheduling.k8s.io/priorityclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list node.k8s.io/runtimeclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/mutatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/validatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.093s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000ff3040\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/volumeattachments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiextensions.k8s.io/customresourcedefinitions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list policy/poddisruptionbudgets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list autoscaling/horizontalpodautoscalers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list coordination.k8s.io/leases: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list discovery.k8s.io/endpointslices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list certificates.k8s.io/certificatesigningrequests: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list scheduling.k8s.io/priorityclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list node.k8s.io/runtimeclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/mutatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/validatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100285588732121088","occurred_at":"2026-09-16T18:08:00Z","signature_id":"7f3c6a5d4502814ef937d1139cc649fca2dcf5ac7887cdb4d24e35c02e2550c3","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100188464501755904","occurred_at":"2026-09-16T11:42:04Z","signature_id":"b62e9e3d53137e1d7201b9bf6b9b273592900eb1294c925f09b1fe4ee1b51c90","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","failure_pattern":"private ingress should not be reachable from outside the VNet, but connection succeeded","search_query":"private ingress should not be reachable from outside the VNet, but connection succeeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2663115845539281,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred.  Got:\n    \u003cnil\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred.  Got:\n    \u003cnil\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100273649465430016","occurred_at":"2026-09-16T17:20:33Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252605216198656","occurred_at":"2026-09-16T15:56:55Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":6933,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-r67qjj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000814820\u003e: \n    DNS for route host agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-r67qjj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000814820\u003e: \n    DNS for route host agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-zxgt9.apps.aro.j0p9f5n7c5x3h1g.7jfd.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-z9mjpt\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010173e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-z9mjpt\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010173e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6mtvt.apps.aro.d3y6k1b1b2n4e7e.ea63.j6198656.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252605216198656","occurred_at":"2026-09-16T15:56:55Z","signature_id":"662575cd71f8c3606ecf799e2d76b50f1b3ecb8c083205f994e08e750a8a40a2","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252605216198656","occurred_at":"2026-09-16T15:56:55Z","signature_id":"d9283aaa1d82489c4a0f2f12dd912bceba77f21d6bae60f6a616ab523d7ace51","pr_number":6933,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1d42c7dad9427d57e1b19754ffe767a1fc187725c1c8da2f03ad1f6733ce9b5d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aaaa60\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6mp8vkxcbh5s\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d35dfb81-23c2-4a18-80a3-7d5bf5893728\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d35dfb81-23c2-4a18-80a3-7d5bf5893728\",\n      \"name\": \"d35dfb81-23c2-4a18-80a3-7d5bf5893728\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T10:51:43.295429479Z\",\n      \"endTime\": \"2026-09-16T11:20:49.710317459Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aaaa60\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6mp8vkxcbh5s\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d35dfb81-23c2-4a18-80a3-7d5bf5893728\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d35dfb81-23c2-4a18-80a3-7d5bf5893728\",\n      \"name\": \"d35dfb81-23c2-4a18-80a3-7d5bf5893728\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T10:51:43.295429479Z\",\n      \"endTime\": \"2026-09-16T11:20:49.710317459Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011044c0\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-95l622wn4sm9\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b24629a-f978-4a3d-9d8d-8d5b6776590f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b24629a-f978-4a3d-9d8d-8d5b6776590f\",\n      \"name\": \"1b24629a-f978-4a3d-9d8d-8d5b6776590f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T10:52:02.869364535Z\",\n      \"endTime\": \"2026-09-16T11:21:09.719035401Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011044c0\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-95l622wn4sm9\" to finish creating: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b24629a-f978-4a3d-9d8d-8d5b6776590f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b24629a-f978-4a3d-9d8d-8d5b6776590f\",\n      \"name\": \"1b24629a-f978-4a3d-9d8d-8d5b6776590f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T10:52:02.869364535Z\",\n      \"endTime\": \"2026-09-16T11:21:09.719035401Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"7299032a0750fba48148c45abb91daa5c3ae005efe59b617e7dc255e091f7a66","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"84f288aa90f1963969fa6df3b97105ac35a5dcb0b0f4f283ef7675cb82451eeb","pr_number":6791,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-update-nodes: failed waiting for nodepool=\"np-update-nodes\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-ckch4xg6nxh4\" to finish creating: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e55eeddb-0ead-432e-a110-c696080abf84\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e55eeddb-0ead-432e-a110-c696080abf84\",\n  \"name\": \"e55eeddb-0ead-432e-a110-c696080abf84\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-16T20:39:13.615216045Z\",\n  \"endTime\": \"2026-09-16T20:58:16.977517435Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"node pool creation did not complete before the deadline\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-update-nodes: failed waiting for nodepool=\"np-update-nodes\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-ckch4xg6nxh4\" to finish creating: GET https://rp.j6757888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e55eeddb-0ead-432e-a110-c696080abf84\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e55eeddb-0ead-432e-a110-c696080abf84\",\n  \"name\": \"e55eeddb-0ead-432e-a110-c696080abf84\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-16T20:39:13.615216045Z\",\n  \"endTime\": \"2026-09-16T20:58:16.977517435Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"node pool creation did not complete before the deadline\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100313784206757888","occurred_at":"2026-09-16T20:00:02Z","signature_id":"663948fad2f655a8f86778686b2675b46d2a717e3dc1afdecbbc520028b3d155","pr_number":6945,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f59e236e067df212a87e97ef00e5cb14f86f6db93d160e4c5cd41734c06ea1b3","failure_pattern":"Admin credentials did not return 409 conflict error while cluster is deploying","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:144]: Admin credentials did not return 409 conflict error while cluster is deploying","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:144]: Admin credentials did not return 409 conflict error while cluster is deploying\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:144]: Admin credentials did not return 409 conflict error while cluster is deploying"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100237146764873728","occurred_at":"2026-09-16T14:55:30Z","signature_id":"6628b29b597aa4c3bb50125f87195e6a58208831ab9330e6078d3d1c523d7af7","pr_number":6752,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"377edc68350e3289212c2970b00ba8365c9d949a03379484101a64858ed9715c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message The collection with name [Resources] in provided scope [/dbs/\u003ccosmos-account\u003e/colls/Resources] could not be found.\\\\\\ \\\\ ActivityId: \u003cuuid\u003e, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0; provider Microsoft.DocumentDB","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1}],"full_error_samples":["time=2026-09-16T13:31:49.803Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-09-16T13:31:51.360Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-09-16T13:31:52.622Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=6a654e03e0d8cd3ba2dcf4b7ca7ea7b9161da4e2734cccce092b3f0e303e2fe9 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6373376-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6a654e03e0d8cd3ba2dcf4b7ca7ea7b9161da4e2734cccce092b3f0e303e2fe9\ntime=2026-09-16T13:40:24.258Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6373376-svc/providers/Microsoft.Resources/deployments/6a654e03e0d8cd3ba2dcf4b7ca7ea7b9161da4e2734cccce092b3f0e303e2fe9/operationStatuses/08584120413735528505\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6373376/providers/Microsoft.Resources/deployments/rp_cosmos_db\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6373376/providers/Microsoft.DocumentDB/databaseAccounts/arohcpci01-rp-j6373376/sqlRoleAssignments/3a943d25-7e89-52ef-bdd2-d3814561859c\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The collection with name [Resources] in provided scope [/dbs/arohcpci01-rp-j6373376/colls/Resources] could not be found.\\\\\\\\r\\\\\\\\nActivityId: 385582e2-ce0e-4904-a9f2-7469a3089fa3, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100209889606373376","occurred_at":"2026-09-16T13:07:12Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6886,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0008473d183c6e76249427529630789e6d4f9e8f1d5bdbea4b6bd57ae83c8f06","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/svc-kv-xehipxbjxawki' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '\u003ctimestamp\u003e' with correlationId '\u003cuuid\u003e', and will expire at '\u003ctimestamp\u003e' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra-output","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/infra-output","occurrences":1}],"full_error_samples":["time=2026-09-16T08:27:04.451Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output description=\"Step infra-output\\n  Kind: ARM\\n  Template: templates/svc-infra-lookup.bicep\\n  Parameters: configurations/svc-infra-lookup.tmpl.bicepparam\"\ntime=2026-09-16T08:27:04.849Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output\ntime=2026-09-16T08:27:05.446Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output deployment=eaaad5ae26368062d8da5183fba1f1cf527c2665c3e53af22a660fa55c8dc157 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7918592-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Feaaad5ae26368062d8da5183fba1f1cf527c2665c3e53af22a660fa55c8dc157\ntime=2026-09-16T08:27:35.588Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=infra-output err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7918592-svc/providers/Microsoft.Resources/deployments/eaaad5ae26368062d8da5183fba1f1cf527c2665c3e53af22a660fa55c8dc157/operationStatuses/08584120596603628303\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentActive\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/svc-kv-xehipxbjxawki' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '9/16/2026 8:27:06 AM' with correlationId '032c9251-9fc6-4ff2-b8e3-30d15533261b', and will expire at '9/23/2026 8:27:06 AM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132012407918592","occurred_at":"2026-09-16T07:57:44Z","signature_id":"09b1458b46da16205cd209ec402488a728d06d1e258b36845618ba3741f96692","pr_number":6639,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"ef6114015f72fab4d7dec149451d78cb6b761172f511b8fdedd7928043056bed","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ErrCode_InsufficientVCPUQuota; detail message Insufficient vcpu quota requested \u003ccount\u003e, remaining \u003ccount\u003e for family StandardEdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: \u003curl\u003e Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-16T15:02:01.563Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-16T15:02:02.231Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-16T15:02:03.235Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=9b458a858b62ced8b861c50221db722ca7b84af5c0247e7866fc1bf068dbe8ac portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7202944-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9b458a858b62ced8b861c50221db722ca7b84af5c0247e7866fc1bf068dbe8ac\ntime=2026-09-16T15:11:05.045Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.Resources/deployments/9b458a858b62ced8b861c50221db722ca7b84af5c0247e7866fc1bf068dbe8ac/operationStatuses/08584120359627896142\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.Resources/deployments/cluster-oqca7h7ulqwlw\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7202944-mgmt-2/agentPools/userswft3\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 96, remaining 48 for family StandardEdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n                  },\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7202944-mgmt-2/agentPools/userswft2\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 96, remaining 48 for family StandardEdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n                  },\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7202944-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7202944-mgmt-2/agentPools/userswft1\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 96, remaining 48 for family StandardEdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100233505077202944","occurred_at":"2026-09-16T14:41:02Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6980,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a6bab59ff05e72a771d612966dd1e7024c1a3d21427661ca54c8003eb2c7e563","failure_pattern":"ERROR CODE: DeploymentFailed; detail code GetMSICredentialError; detail message Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1}],"full_error_samples":["time=2026-09-16T12:27:55.094Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-09-16T12:27:56.479Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-09-16T12:27:57.833Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=b489aeac3f7b0efc7498443b42a85f2f65ee62ab4f82355239f486d3dbc78483 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5020544-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fb489aeac3f7b0efc7498443b42a85f2f65ee62ab4f82355239f486d3dbc78483\ntime=2026-09-16T12:31:28.399Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5020544-svc/providers/Microsoft.Resources/deployments/b489aeac3f7b0efc7498443b42a85f2f65ee62ab4f82355239f486d3dbc78483/operationStatuses/08584120452084424414\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5020544-svc/providers/Microsoft.Resources/deployments/cluster-6sk23df5akobi\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5020544-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j5020544-svc\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"GetMSICredentialError\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Get managed identity credential failed. Details: Category: ClientError; Code: NotFound; SubCode: NotFound; Message: unexpected response from MSI data plane, length of returned certificate: 0; Dependency: MSI Credential; AKSTeam: RP\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100192568775020544","occurred_at":"2026-09-16T11:58:22Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6964,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"fe24c8b4b28d3d2b1937803a92704c44ec574f3d1c3b740ec6d97282a1e18d39","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00230a300\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ql5j7w6fx6vn\" to finish creating: GET https://rp.j6364288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5fde05ba-68c2-437a-84ea-82765b356728\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5fde05ba-68c2-437a-84ea-82765b356728\",\n      \"name\": \"5fde05ba-68c2-437a-84ea-82765b356728\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T11:04:09.351110934Z\",\n      \"endTime\": \"2026-09-16T11:04:15.302518412Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00230a300\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ql5j7w6fx6vn\" to finish creating: GET https://rp.j6364288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5fde05ba-68c2-437a-84ea-82765b356728\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5fde05ba-68c2-437a-84ea-82765b356728\",\n      \"name\": \"5fde05ba-68c2-437a-84ea-82765b356728\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T11:04:09.351110934Z\",\n      \"endTime\": \"2026-09-16T11:04:15.302518412Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100164075236364288","occurred_at":"2026-09-16T10:05:09Z","signature_id":"59512808dbcb20b06030c9fde07924554878fee722f13b2a06c73ba1d94bd2f9","pr_number":6688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6ed58550ec6b76673ce00588c8bebee5ecb6ea4c774ba1ceae5e31e8e80cd829","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000edc320\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-d77hgbbbw6rf\" to finish creating: GET https://rp.j6676096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74e66356-25b3-4d6f-95af-f1292008b6fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74e66356-25b3-4d6f-95af-f1292008b6fd\",\n      \"name\": \"74e66356-25b3-4d6f-95af-f1292008b6fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:47:22.69341748Z\",\n      \"endTime\": \"2026-09-16T21:16:24.116227211Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 6m20s ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000edc320\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-d77hgbbbw6rf\" to finish creating: GET https://rp.j6676096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74e66356-25b3-4d6f-95af-f1292008b6fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74e66356-25b3-4d6f-95af-f1292008b6fd\",\n      \"name\": \"74e66356-25b3-4d6f-95af-f1292008b6fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:47:22.69341748Z\",\n      \"endTime\": \"2026-09-16T21:16:24.116227211Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 6m20s ago; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100314659356676096","occurred_at":"2026-09-16T20:03:31Z","signature_id":"ec03acc423d7d391cd0db46f80f6e282dd4d144bd87644aded93eda8c1bf31c4","pr_number":6969,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c2687040eef152546e303fe3ce6da2f92a92db4c9a3989bfdcd2df869f68897f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-lzpm5g\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d707c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-lzpm5g: failed waiting for cluster=\"cp-ystream-upgrade-4-22-lzpm5g\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-v5f9s54xdgwx\" to finish creating: GET https://rp.j9719424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cff0006a-d905-424f-9efd-d8db75f2fd05\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cff0006a-d905-424f-9efd-d8db75f2fd05\",\n      \"name\": \"cff0006a-d905-424f-9efd-d8db75f2fd05\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:49:09.853286386Z\",\n      \"endTime\": \"2026-09-16T21:18:11.008528958Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 13m4s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-lzpm5g\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d707c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-lzpm5g: failed waiting for cluster=\"cp-ystream-upgrade-4-22-lzpm5g\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-v5f9s54xdgwx\" to finish creating: GET https://rp.j9719424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cff0006a-d905-424f-9efd-d8db75f2fd05\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cff0006a-d905-424f-9efd-d8db75f2fd05\",\n      \"name\": \"cff0006a-d905-424f-9efd-d8db75f2fd05\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:49:09.853286386Z\",\n      \"endTime\": \"2026-09-16T21:18:11.008528958Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 13m4s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309343839719424","occurred_at":"2026-09-16T19:42:24Z","signature_id":"af5ff2a9735a8e441c18465437503752dd1901dae01a546fbec14adb73b0c7e5","pr_number":6403,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5d59b4e4feb4fd290a5c1b398014539407c812b82baa8f933608971503b0bc0d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8a120\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-8prbhvsvlkq9\" to finish creating: GET https://rp.j9922816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b148abb4-8011-43a5-acf9-7d2f1f124b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b148abb4-8011-43a5-acf9-7d2f1f124b53\",\n      \"name\": \"b148abb4-8011-43a5-acf9-7d2f1f124b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:56:50.517658972Z\",\n      \"endTime\": \"2026-09-16T21:25:54.975318736Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8a120\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-8prbhvsvlkq9\" to finish creating: GET https://rp.j9922816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b148abb4-8011-43a5-acf9-7d2f1f124b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b148abb4-8011-43a5-acf9-7d2f1f124b53\",\n      \"name\": \"b148abb4-8011-43a5-acf9-7d2f1f124b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T20:56:50.517658972Z\",\n      \"endTime\": \"2026-09-16T21:25:54.975318736Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100309676229922816","occurred_at":"2026-09-16T19:43:43Z","signature_id":"a48d7255fe4f2e537c6260d9fc791f5067ce6859f69937f424d6c26f5bc86ca9","pr_number":6894,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"676e68f8a68616ac8c57dc0fe075f4a0d00c211dff572a92461d02984b57a771","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098aa40\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-6th5jd8nbg5s\" to finish creating: GET https://rp.j2424320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd1a9666-9126-4a24-9b9a-2799ae9c069b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd1a9666-9126-4a24-9b9a-2799ae9c069b\",\n      \"name\": \"bd1a9666-9126-4a24-9b9a-2799ae9c069b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T13:57:52.148119457Z\",\n      \"endTime\": \"2026-09-16T14:26:53.36802381Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, community-operators-catalog, hosted-cluster-config-operator, catalog-operator, redhat-operators-catalog, ingress-operator, cluster-image-registry-operator, openshift-route-controller-manager, cluster-node-tuning-operator, dns-operator, redhat-marketplace-catalog, certified-operators-catalog, cluster-network-operator, packageserver, openshift-controller-manager, cluster-policy-controller, cluster-version-operator, cluster-storage-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098aa40\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-6th5jd8nbg5s\" to finish creating: GET https://rp.j2424320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd1a9666-9126-4a24-9b9a-2799ae9c069b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd1a9666-9126-4a24-9b9a-2799ae9c069b\",\n      \"name\": \"bd1a9666-9126-4a24-9b9a-2799ae9c069b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T13:57:52.148119457Z\",\n      \"endTime\": \"2026-09-16T14:26:53.36802381Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, community-operators-catalog, hosted-cluster-config-operator, catalog-operator, redhat-operators-catalog, ingress-operator, cluster-image-registry-operator, openshift-route-controller-manager, cluster-node-tuning-operator, dns-operator, redhat-marketplace-catalog, certified-operators-catalog, cluster-network-operator, packageserver, openshift-controller-manager, cluster-policy-controller, cluster-version-operator, cluster-storage-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100212360202424320","occurred_at":"2026-09-16T13:16:57Z","signature_id":"fec53f2216f9f1f9da9e1788f3642b62d62882fc08fe207f753fbff13e9390bf","pr_number":6894,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a9f77fe91a65ae001825b7e195168037cff1cc1fd2f1b2a967813a5f71db56d2","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f92288\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-djgr48qrs9vr\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\",\n      \"name\": \"a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:36:21.892022588Z\",\n      \"endTime\": \"2026-09-16T10:00:22.50023585Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss12t13cvqgvb69duuok3m0c1if4g29 still exists (deletion dispatched at 2026-09-16T09:36:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f92288\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-djgr48qrs9vr\" to finish deleting: GET https://rp.j5539456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\",\n      \"name\": \"a17df9ed-a5b1-4a4c-b59d-0c40cfdd60e7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T09:36:21.892022588Z\",\n      \"endTime\": \"2026-09-16T10:00:22.50023585Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss12t13cvqgvb69duuok3m0c1if4g29 still exists (deletion dispatched at 2026-09-16T09:36:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100141632635539456","occurred_at":"2026-09-16T08:35:58Z","signature_id":"f25ee5738903cfa5e80de14eb219fe7a6033dded2da21704c137087674937bf8","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b2d9bdf10bd756a41af895df7f16c38d9b8ca7729eae2184e9c4375fba7db5fb","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012b4030\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-sk5lk2xqrw7v\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/234e26dc-5c76-4335-9749-4a1c49aae81e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/234e26dc-5c76-4335-9749-4a1c49aae81e\",\n      \"name\": \"234e26dc-5c76-4335-9749-4a1c49aae81e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:00:40.136241522Z\",\n      \"endTime\": \"2026-09-16T17:24:42.83613595Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7is141l1pe3gtmk90b2cej7ltp3gn still exists (deletion dispatched at 2026-09-16T17:00:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012b4030\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-sk5lk2xqrw7v\" to finish deleting: GET https://rp.j4807552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/234e26dc-5c76-4335-9749-4a1c49aae81e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/234e26dc-5c76-4335-9749-4a1c49aae81e\",\n      \"name\": \"234e26dc-5c76-4335-9749-4a1c49aae81e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T17:00:40.136241522Z\",\n      \"endTime\": \"2026-09-16T17:24:42.83613595Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ss7is141l1pe3gtmk90b2cej7ltp3gn still exists (deletion dispatched at 2026-09-16T17:00:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"5cf9c85c408d35be4deef4e7c087c454f1070d1509c8652940e0ebeb5471037c","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f3bc88c6c8876f1c2ea5668dd924adc7c81d6e0349b9850342e67442ed3fe914","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00108c078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mzfrqt8tf9wq\" to finish deleting: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\",\n      \"name\": \"e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T11:18:42.744808437Z\",\n      \"endTime\": \"2026-09-16T11:42:49.617012035Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00108c078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mzfrqt8tf9wq\" to finish deleting: GET https://rp.j5236352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\",\n      \"name\": \"e9f01e6a-fcf9-4bbb-aeeb-61ee6e0ac502\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T11:18:42.744808437Z\",\n      \"endTime\": \"2026-09-16T11:42:49.617012035Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100161923835236352","occurred_at":"2026-09-16T09:56:36Z","signature_id":"874e6ea76f2ba4acc89e73ac761bf0c81c038b75f11db73023de66c025986ab7","pr_number":6791,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f98a5615cab85c5989e6a0ac742f663ddff170e09cb4f716bf9a5978a279cf45","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to get pod logs for pod fips-check-psxxs in namespace e2e-fips-test-\u003cid\u003e on node \u003cnode\u003e after retries: an error on the server (\"unknown\") has prevented the request from succeeding","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5: failed to get pod logs for pod fips-check-psxxs in namespace e2e-fips-test-rx7hl on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-psxxs)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc002923158\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5: failed to get pod logs for pod fips-check-psxxs in namespace e2e-fips-test-rx7hl on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-psxxs)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc002923158\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5: failed to get pod logs for pod fips-check-psxxs in namespace e2e-fips-test-rx7hl on node o2d9i5v3g7d8o6s-np-1-9rk5g-gzzc5 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-psxxs)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100323463913803776","occurred_at":"2026-09-16T20:38:26Z","signature_id":"64d1fa05ea54883aa2363d63f6b4eef64e006023913f4dea31d5e86c98373938","pr_number":6805,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc002e66048\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc002e66048\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-znw56.apps.aro.agg-cluster.fd32.j2846464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6966/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100263412582846464","occurred_at":"2026-09-16T16:39:53Z","signature_id":"dcd0bfb37a59db9bd0b774c26b5a47b2e4002b0f9d6d7d9a6292d0c9dca75bea","pr_number":6966,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T15:28:20Z\n  Ended:       2026-09-16T15:40:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j8680064-svc\", component=\"backend\", name=\"dataplaneoidcfederation\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue dataplaneoidcfederation has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6983/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100226936398680064","occurred_at":"2026-09-16T14:14:56Z","signature_id":"9efd97c6c35dce815008610c2dc8e01945d6be74819a30c6714b0f4b7f9ad9da","pr_number":6983,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a89f0d2edcd262580a6fd914503ced16ad52ae763f2c2255b4f5af55d375ecd3","failure_pattern":"alert [svc] KubeContainerWaiting fired","search_query":"alert [svc] KubeContainerWaiting fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerWaiting does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerWaiting does not fire","occurrences":1}],"full_error_samples":["alert fired 5 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:30:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"secrets-store\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-cwxxr\", region=\"canadacentral\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-cwxxr in namespace kube-system on container secrets-store has been in waiting state for longer than 1 hour.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:30:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"liveness-probe\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-cwxxr\", region=\"canadacentral\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-cwxxr in namespace kube-system on container liveness-probe has been in waiting state for longer than 1 hour.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:30:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"node-driver-registrar\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-cwxxr\", region=\"canadacentral\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-cwxxr in namespace kube-system on container node-driver-registrar has been in waiting state for longer than 1 hour.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:30:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", namespace=\"kube-system\", pod=\"ama-metrics-node-xfmjp\", region=\"canadacentral\", severity=\"warning\"\n  Description: pod/ama-metrics-node-xfmjp in namespace kube-system on container prometheus-collector has been in waiting state for longer than 1 hour.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:30:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"addon-token-adapter\", namespace=\"kube-system\", pod=\"ama-metrics-node-xfmjp\", region=\"canadacentral\", severity=\"warning\"\n  Description: pod/ama-metrics-node-xfmjp in namespace kube-system on container addon-token-adapter has been in waiting state for longer than 1 hour."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"97461ac292fa9fba61fa81d505ebf8a88d3f96ffd9a07f7d644245f3dd5aa26b","pr_number":6982,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d6c6b690ca60846dd98e464313d6a9672b9abd8744604edd915627eb68203f9d","failure_pattern":"alert [svc] KubeNodeNotReady fired","search_query":"alert [svc] KubeNodeNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeNotReady does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeNotReady does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:00:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"prometheus\", node=\"aks-system-28181016-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-system-28181016-vmss000000 has been unready for more than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:00:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j6091136-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.38:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6091136/providers/microsoft.monitor/accounts/services-j6091136\", namespace=\"prometheus\", node=\"aks-system-28181016-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-qxmdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-system-28181016-vmss000000 has been unready for more than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100248871216091136","occurred_at":"2026-09-16T15:42:06Z","signature_id":"e58f291d243075248b87935d2709ed85f1b36fdb31f78f88aaae185cc551feae","pr_number":6982,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":1}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:54:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j4807552-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.89:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4807552/providers/microsoft.monitor/accounts/services-j4807552\", namespace=\"klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-zvqnn\", prometheus=\"prometheus/prometheus\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease work-manager in namespace klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn on cluster ci01-j4807552-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:54:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j4807552-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.89:8080\", job=\"kube-state-metrics\", lease=\"config-policy-controller\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4807552/providers/microsoft.monitor/accounts/services-j4807552\", namespace=\"klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-zvqnn\", prometheus=\"prometheus/prometheus\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease config-policy-controller in namespace klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn on cluster ci01-j4807552-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-16T17:54:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j4807552-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.89:8080\", job=\"kube-state-metrics\", lease=\"governance-policy-framework\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4807552/providers/microsoft.monitor/accounts/services-j4807552\", namespace=\"klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-zvqnn\", prometheus=\"prometheus/prometheus\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Leader election lease governance-policy-framework in namespace klusterlet-2ss7is141l1pe3gtmk90b2cej7ltp3gn on cluster ci01-j4807552-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252330694807552","occurred_at":"2026-09-16T15:55:51Z","signature_id":"ed2b24c463c24323e023a9ff56bf6535b99fca0fd95a6e6a5f3b0c1f68b41036","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d89bec3ea75e80fed983a698bbef1fbb78c33e77e634e6400cab6b214e710fe5","failure_pattern":"alert [svc] userJourneyFrontendLatencyP991h5m fired","search_query":"alert [svc] userJourneyFrontendLatencyP991h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP991h5m does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatencyP991h5m does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-16T09:39:29Z\n  Ended:       2026-09-16T09:50:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatencyP991h5m\", cluster=\"ci01-j3290112-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3290112/providers/microsoft.monitor/accounts/services-j3290112\", severity=\"3\", short_window=\"5m\", slo=\"frontend-latency\"\n  Description: Frontend p99 latency on cluster ci01-j3290112-svc is above 5s on both the 1h and 5m windows (severe miss of the p99 \u003c 1s ARM sync SLO).\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-16T10:10:33Z\n  Ended:       2026-09-16T10:17:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatencyP991h5m\", cluster=\"ci01-j3290112-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3290112/providers/microsoft.monitor/accounts/services-j3290112\", severity=\"3\", short_window=\"5m\", slo=\"frontend-latency\"\n  Description: Frontend p99 latency on cluster ci01-j3290112-svc is above 5s on both the 1h and 5m windows (severe miss of the p99 \u003c 1s ARM sync SLO)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132038643290112","occurred_at":"2026-09-16T07:57:51Z","signature_id":"13f9b4b06b1b9850ff1976e03ab24843f0b045626fd218be7150c7dace6ffbef","pr_number":6503,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e9ea5f54afe3e98f1a64398661238c6fc0e342dcd1119b0d656cfccf60db80ec","failure_pattern":"breakglass session kubeconfig not ready: timeout waiting for session to become ready; HostedControlPlane exists but is not ready","search_query":"failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-wlfqkcdjh9rm/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-lbdxk/kubeconfig: timeout waiting for session to become ready (last status: {\"HostedControlPlaneAvailable\":{\"message\":\"HostedControlPlane exists but is not ready\",\"reason\":\"HostedControlPlaneNotReady\",\"status\":\"False\"},\"status\":\"Session is not ready\"})","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:185]: failed to create SRE breakglass credentials with aro-sre-pso permissions\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a7f820\u003e: \n    failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-wlfqkcdjh9rm/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-lbdxk/kubeconfig: timeout waiting for session to become ready (last status: {\"HostedControlPlaneAvailable\":{\"message\":\"HostedControlPlane exists but is not ready\",\"reason\":\"HostedControlPlaneNotReady\",\"status\":\"False\"},\"status\":\"Session is not ready\"})\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:185]: failed to create SRE breakglass credentials with aro-sre-pso permissions\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a7f820\u003e: \n    failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-wlfqkcdjh9rm/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-lbdxk/kubeconfig: timeout waiting for session to become ready (last status: {\"HostedControlPlaneAvailable\":{\"message\":\"HostedControlPlane exists but is not ready\",\"reason\":\"HostedControlPlaneNotReady\",\"status\":\"False\"},\"status\":\"Session is not ready\"})\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100153419804184576","occurred_at":"2026-09-16T09:22:48Z","signature_id":"6c4e73f54e8d63150bdff29f74282ba9a9e2ff1e60f7bd58d45e6cacc3dd7869","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fac5ab822da5a4db89037217494583b4069bba01be61d8f3a5122bc721b483e9","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"0IciLMVmLbqimBoU8f5q6QBEpmWjuJNgDzWaCM2v9MM=\"","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"0IciLMVmLbqimBoU8f5q6QBEpmWjuJNgDzWaCM2v9MM=\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001ffe8a0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"0IciLMVmLbqimBoU8f5q6QBEpmWjuJNgDzWaCM2v9MM=\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001ffe8a0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: 404 urn:ietf:params:acme:error:malformed: checking for existing account :: no registrations with public key sha256 \"0IciLMVmLbqimBoU8f5q6QBEpmWjuJNgDzWaCM2v9MM=\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100252538870697984","occurred_at":"2026-09-16T15:56:40Z","signature_id":"b7562eb1317f094fe7e5e1b3fd0e3859a05304435b1ff984fb6d716f402b4c7c","pr_number":6791,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gn4q6q4pqf7b\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0011eef40\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gn4q6q4pqf7b\" to finish creating: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c302cb08-5816-4f3c-a786-daa45a7d6406\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c302cb08-5816-4f3c-a786-daa45a7d6406\",\n      \"name\": \"c302cb08-5816-4f3c-a786-daa45a7d6406\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:12:22.418838081Z\",\n      \"endTime\": \"2026-09-16T15:13:01.450563178Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0011eef40\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gn4q6q4pqf7b\" to finish creating: GET https://rp.j4331648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c302cb08-5816-4f3c-a786-daa45a7d6406\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c302cb08-5816-4f3c-a786-daa45a7d6406\",\n      \"name\": \"c302cb08-5816-4f3c-a786-daa45a7d6406\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T15:12:22.418838081Z\",\n      \"endTime\": \"2026-09-16T15:13:01.450563178Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222840434331648","occurred_at":"2026-09-16T13:58:40Z","signature_id":"162026bfd655fc4db32e7049c532e8d81f8bd34a8e4e6625145cd5a31c7568a3","pr_number":6976,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e45f8e323244cd43106961bb8120aab24b3a867987bf8529ce7b7b4509a092cc","failure_pattern":"failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/18 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/18\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kubelet-ds (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-16T09:03:26.822Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 description=\"Step kubelet-ds\\n Kind: Helm\\n\"\ntime=2026-09-16T09:03:27.157Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 values=\"map[enabled:true kubeletParametersImage:mcr.microsoft.com/aks/command/runtime@sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23]\"\ntime=2026-09-16T09:03:27.157Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-09-16T09:03:27.284Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 namespace=kubelet-ds\ntime=2026-09-16T09:03:27.475Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 dryRun=false\ntime=2026-09-16T09:03:27.475Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-09-16T09:03:27.530Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2\ntime=2026-09-16T09:08:28.583Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/18\\ncontext deadline exceeded\"\ntime=2026-09-16T09:08:28.583Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kubelet-ds stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/kube-system/set-kubelet-parameters-for-scale not ready. status: InProgress, message: Available: 0/18\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100132003708932096","occurred_at":"2026-09-16T07:57:42Z","signature_id":"4592cebb946834bf2f5597cc8f68b2f39cdbe688bce563e773bde0e8a997725a","pr_number":6009,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"662e49bfb814d1960ee5fb9e525e6ffab8495cd359fc51176893943560eee29a","failure_pattern":"node pool \"\u003cnodepool\u003e\" should have exactly one active version after initial creation","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:144]: node pool \"np-1\" should have exactly one active version after initial creation","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:144]: node pool \"np-1\" should have exactly one active version after initial creation\nExpected\n    \u003c[]*armredhatopenshifthcp.NodePoolActiveVersion | len:0, cap:0\u003e: ...\nto have length 1\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:144]: node pool \"np-1\" should have exactly one active version after initial creation\nExpected\n    \u003c[]*armredhatopenshifthcp.NodePoolActiveVersion | len:0, cap:0\u003e: ...\nto have length 1"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100222891458039808","occurred_at":"2026-09-16T13:58:52Z","signature_id":"4c1a786ccdf5f4a6714330ab65d6eaf1faaac9d21fed54613d33cde0056d6e52","pr_number":6952,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wngrxcmwz6zh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-wngrxcmwz6zh, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.13315579227696406,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009db7a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wngrxcmwz6zh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-wngrxcmwz6zh, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009db7a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-wngrxcmwz6zh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-wngrxcmwz6zh, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100229912697769984","occurred_at":"2026-09-16T14:26:41Z","signature_id":"f366c8a9d3b487352e0d087e494e88beecbef611270fb1a6b15974fec8f76394","pr_number":6364,"after_last_push_of_merged_pr":false}]}]},{"environment":"int","summary":{"total_runs":27,"failed_runs":10,"raw_occurrences":10,"matched_occurrences":1,"runs_affected":1},"rows":[{"environment":"int","failure_pattern_id":"a52d3f63256add31d93e084d23258696e3ce9f667510ec06dc509046b07f2f14","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.7037037037037037,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD","STG"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000df6cd8\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-pfttlc9b26s6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\",\n      \"name\": \"ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T22:57:32.075098968Z\",\n      \"endTime\": \"2026-09-16T23:16:32.974740274Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:204]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000df6cd8\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-pfttlc9b26s6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\",\n      \"name\": \"ff088bfb-6289-4eb8-a3c7-2bd5dc2e8821\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T22:57:32.075098968Z\",\n      \"endTime\": \"2026-09-16T23:16:32.974740274Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100351631508901888","occurred_at":"2026-09-16T22:30:19Z","signature_id":"f7210bc017800c8d3ce9292e2b300557e56751eedb6338b78091f529330d0435","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"prod","summary":{"total_runs":22,"failed_runs":22,"raw_occurrences":29,"matched_occurrences":9,"runs_affected":2},"rows":[{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.545454545454546,"anchor_occurrences":5,"after_last_push_count":5,"also_in":["STG"],"trend_counts":[0,0,0,0,0,0,0,5],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001d82db0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-d2hkj9sg5shk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001d82db0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-d2hkj9sg5shk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000fd73e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-dqvpfm666hzz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000fd73e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-dqvpfm666hzz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002c97050\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8h5dwhv96zkm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002c97050\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8h5dwhv96zkm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"494d690c10401d515d7d97bb1b43282cd5512bedf4a8cf0e93991a312d0f71e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"5f0d2310ed06414a49e9f1ddd637c2a2516af067a00a8168124b28d160b1f9b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"a91c26d043927cb825f419af86a21ed7c54daa9e6da09ecdbe0fab7af0a8ab85","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"b2ee8d4e7b429ad69336ec70ed0fafeb7a58d728ccf645b31d75121fd78def37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"e2f2d3555780f9a4be9e4c49fe522f13754292efbc3c1299543a5305777d3351","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.545454545454546,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","INT","STG"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3cd80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-b7c4p2nvk6d6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0259344c-f395-4714-abc8-4abafaaf707d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0259344c-f395-4714-abc8-4abafaaf707d\",\n      \"name\": \"0259344c-f395-4714-abc8-4abafaaf707d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T23:18:34.031156052Z\",\n      \"endTime\": \"2026-09-16T23:37:37.814154784Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3cd80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-b7c4p2nvk6d6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0259344c-f395-4714-abc8-4abafaaf707d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0259344c-f395-4714-abc8-4abafaaf707d\",\n      \"name\": \"0259344c-f395-4714-abc8-4abafaaf707d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T23:18:34.031156052Z\",\n      \"endTime\": \"2026-09-16T23:37:37.814154784Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"aae2457e0f58577b233ddb5159361da1324cc74ad01ffe504591fd6f9131c0dd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.545454545454546,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d85560\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-jzkmnxzs9c52\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jzkmnxzs9c52/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d85560\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-jzkmnxzs9c52\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jzkmnxzs9c52/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"70af6c4c5a716b4a2eb8db0539e648de82b2496a872a99a8c6a98074fd41d4c3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.545454545454546,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00152ab10\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00152ab10\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100359142039359488","occurred_at":"2026-09-16T23:00:16Z","signature_id":"f5ce501d2dd69dcdea5a25282314ce7ff4dede1a06ceeb918b7e36499f9665b6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b57f3261aa83aadd9ceb7988b00830acc29894d12dc4f5f19baa862276967565","failure_pattern":"cluster operators not available: console (no Available condition), dns (Available=False, Progressing=True, Degraded=True, Upgradeable=True), image-registry (Available=False, Progressing=True, Degraded=True), ingress (Available=False, Progressing=True","search_query":"cluster operators not available: console (no Available condition), dns (Available=False, Progressing=True, Degraded=True, Upgradeable=True), image-registry (Available=False, Progressing=True, Degraded=True), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False), monitoring (no Available condition), openshift-samples (no Available condition), storage (Degraded=False, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown)","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.545454545454546,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc002a4e018\u003e: \n    context deadline exceeded\n    cluster operators not available: console (no Available condition), dns (Available=False, Progressing=True, Degraded=True, Upgradeable=True), image-registry (Available=False, Progressing=True, Degraded=True), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False), monitoring (no Available condition), openshift-samples (no Available condition), storage (Degraded=False, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc002a4e018\u003e: \n    context deadline exceeded\n    cluster operators not available: console (no Available condition), dns (Available=False, Progressing=True, Degraded=True, Upgradeable=True), image-registry (Available=False, Progressing=True, Degraded=True), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False), monitoring (no Available condition), openshift-samples (no Available condition), storage (Degraded=False, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100173746357669888","occurred_at":"2026-09-16T10:43:35Z","signature_id":"cfd34eb004cc5a7cf7577f79d571b368c1d8dfc53041aa652f9465c361a3b4a0","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":24,"failed_runs":23,"raw_occurrences":34,"matched_occurrences":12,"runs_affected":1},"rows":[{"environment":"stg","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":5,"after_last_push_count":5,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,0,0,5],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001d4bc80\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-56f2xp2w67w8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001d4bc80\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-56f2xp2w67w8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012fd950\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-57279mxs72dc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012fd950\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-57279mxs72dc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f2fd40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-g8sp2mm6ks67/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f2fd40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-g8sp2mm6ks67/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"3799930be8939ca48534a28b262ff33328138426e7cec1b9fb2793e6f4a95899","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"474ef928e4bfaa163ff38e770c6435689354cdc8821b9a05ec4272d8ad60c351","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"7a42cc1723b1b8bf9843f1aa12dd8baec4448f6917d7f084b3a4b154baa63afa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"8be57f60e7cd91c124c72d6d01098c6e1b8b853355d51b79f6b1bd55c240efdb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"bd4c07c42b25870c0f1baaecfa45c3c668afd2f7347cf758e572515c16e421fd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,0,4],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004d4640\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-c8fd7f\" in resourcegroup=\"rg-np-version-upgrade-c8fd7f-kdnr5znbw57r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-c8fd7f-kdnr5znbw57r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-c8fd7f/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004d4640\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-c8fd7f\" in resourcegroup=\"rg-np-version-upgrade-c8fd7f-kdnr5znbw57r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-c8fd7f-kdnr5znbw57r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-c8fd7f/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002c1a920\u003e: \n    failed to create NodePool np-pbnkzl: failed starting nodepool creation \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\" in resourcegroup=\"rg-nightly-4-20-6c9ns9nbgrzm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-6c9ns9nbgrzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-pbnkzl/nodePools/np-pbnkzl\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002c1a920\u003e: \n    failed to create NodePool np-pbnkzl: failed starting nodepool creation \"np-pbnkzl\" for cluster \"cluster-nightly-4-20-pbnkzl\" in resourcegroup=\"rg-nightly-4-20-6c9ns9nbgrzm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-6c9ns9nbgrzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-pbnkzl/nodePools/np-pbnkzl\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000facdc0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-m58t8j\" resourcegroup=\"rg-np-dg-minor-m58t8j-trxh29mwfdlb\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-m58t8j-trxh29mwfdlb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-m58t8j/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-15-135250\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000facdc0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-m58t8j\" resourcegroup=\"rg-np-dg-minor-m58t8j-trxh29mwfdlb\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-m58t8j-trxh29mwfdlb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-m58t8j/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-15-135250\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"3d8d555b219d0ed8c9661c956fba32429ade34240dfb89932e7b6f838ec71e56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"51c0a7fe19269071bf4a86bc72b032196ba0e6e4e142e092ab1f3aca5f282e06","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"a3bcc0cf85a56ec12c072c420e0931c744e3a86ca315e7af1b9dd0b787fa5b91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"c259628d57d4675c02d0130322edaa375c878722b676fb3290dbafba51543aa4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","INT","PROD"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-krkxx8\" for cluster \"cluster-nightly-4-23-krkxx8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00121ee40\u003e: \n    failed to create NodePool np-krkxx8: failed waiting for nodepool=\"np-krkxx8\" for cluster \"cluster-nightly-4-23-krkxx8\" in resourcegroup=\"rg-nightly-4-23-7l9rdmzm7f2z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\",\n      \"name\": \"4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T23:37:25.694607859Z\",\n      \"endTime\": \"2026-09-16T23:56:27.737971571Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-krkxx8\" for cluster \"cluster-nightly-4-23-krkxx8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00121ee40\u003e: \n    failed to create NodePool np-krkxx8: failed waiting for nodepool=\"np-krkxx8\" for cluster \"cluster-nightly-4-23-krkxx8\" in resourcegroup=\"rg-nightly-4-23-7l9rdmzm7f2z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\",\n      \"name\": \"4e59a6ba-7f62-48bf-8e29-7e6ddefd08f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-16T23:37:25.694607859Z\",\n      \"endTime\": \"2026-09-16T23:56:27.737971571Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"18ab22781017136dd055b840d502183c7edf1caa41f67dcadf9fd0a7857c8081","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"5c1e65a74237eb97d61c283de75bcde39935ce10b1226f1bc067c35315e03a3d","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b5a80\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-2wnkdzqz6d9w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2wnkdzqz6d9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b5a80\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-2wnkdzqz6d9w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2wnkdzqz6d9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"5c2b3fa28b82ccf14825b20d7135a7cb3d0c7f8bdbad597c1b9eba4c000e0333","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.166666666666667,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-09..2026-09-16 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0016e5a88\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0016e5a88\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100359142060331008","occurred_at":"2026-09-16T23:00:16Z","signature_id":"77617baa5bf45330c8ac3ac9717b66793245c58ebcc17c8a3e357cf43d6f0bf5","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}