{"meta":{"start_date":"2026-08-13","end_date":"2026-08-13","start_at":"2026-08-13T00:00:00Z","end_at":"2026-08-14T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T03:05:28Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":9,"failed_runs":6,"raw_occurrences":81,"matched_occurrences":81,"runs_affected":6},"rows":[{"environment":"prod","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":44,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":44,"after_last_push_count":77,"trend_counts":[0,0,0,0,0,0,44],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-03"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-06T12:27:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:741]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d92a80\u003e: \n    failed to create HCP cluster np-dg-minor-pghz8v: failed waiting for cluster=\"np-dg-minor-pghz8v\" in resourcegroup=\"rg-np-dg-minor-pghz8v-lrw5sgmh8jl2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1fe2ebf3-2ba2-4f3a-9716-d209bd154df6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1fe2ebf3-2ba2-4f3a-9716-d209bd154df6\",\n      \"name\": \"1fe2ebf3-2ba2-4f3a-9716-d209bd154df6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T14:17:58.32975278Z\",\n      \"endTime\": \"2026-08-13T14:37:00.365479085Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:741]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d92a80\u003e: \n    failed to create HCP cluster np-dg-minor-pghz8v: failed waiting for cluster=\"np-dg-minor-pghz8v\" in resourcegroup=\"rg-np-dg-minor-pghz8v-lrw5sgmh8jl2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1fe2ebf3-2ba2-4f3a-9716-d209bd154df6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1fe2ebf3-2ba2-4f3a-9716-d209bd154df6\",\n      \"name\": \"1fe2ebf3-2ba2-4f3a-9716-d209bd154df6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T14:17:58.32975278Z\",\n      \"endTime\": \"2026-08-13T14:37:00.365479085Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001f86f00\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-fst58bwgz7np\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c2157648-4336-474c-bfaf-52608e28607d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c2157648-4336-474c-bfaf-52608e28607d\",\n      \"name\": \"c2157648-4336-474c-bfaf-52608e28607d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T13:26:57.877053143Z\",\n      \"endTime\": \"2026-08-13T13:45:57.039213286Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001f86f00\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-fst58bwgz7np\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c2157648-4336-474c-bfaf-52608e28607d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c2157648-4336-474c-bfaf-52608e28607d\",\n      \"name\": \"c2157648-4336-474c-bfaf-52608e28607d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T13:26:57.877053143Z\",\n      \"endTime\": \"2026-08-13T13:45:57.039213286Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:462]: failed to create HCP cluster np-skip-minor-84bfxc at version 4.22.9\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005af6e0\u003e: \n    failed to create HCP cluster np-skip-minor-84bfxc: failed waiting for cluster=\"np-skip-minor-84bfxc\" in resourcegroup=\"rg-np-skip-minor-84bfxc-v5vcr2r2wlh6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/d88d3ae3-e884-4645-a7b9-3c8d769f6966\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/d88d3ae3-e884-4645-a7b9-3c8d769f6966\",\n      \"name\": \"d88d3ae3-e884-4645-a7b9-3c8d769f6966\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T14:17:58.694323926Z\",\n      \"endTime\": \"2026-08-13T14:37:01.46212293Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:462]: failed to create HCP cluster np-skip-minor-84bfxc at version 4.22.9\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005af6e0\u003e: \n    failed to create HCP cluster np-skip-minor-84bfxc: failed waiting for cluster=\"np-skip-minor-84bfxc\" in resourcegroup=\"rg-np-skip-minor-84bfxc-v5vcr2r2wlh6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/d88d3ae3-e884-4645-a7b9-3c8d769f6966\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/d88d3ae3-e884-4645-a7b9-3c8d769f6966\",\n      \"name\": \"d88d3ae3-e884-4645-a7b9-3c8d769f6966\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T14:17:58.694323926Z\",\n      \"endTime\": \"2026-08-13T14:37:01.46212293Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"01a31292cd515dc8f519670b243d3abfd4119a9483f99307a8bd96fae20c7525","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"1672b823d5bb91129bcdf629a4bfd1361e098826a6e9cc1f8082529f3f9791ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"1b6bb5ed4c2ea791022ddbaee83504568cf7d9c494953007b9739f644614aa10","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"203b46a994d5d317fdb59bdd7dc81eba37cbecacc4f1ddd570442a328e2be045","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"3c613e199a0bf63f2287947ce3d7d2f24dc1274357038e57b57c080830c2ca68","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"425be683392c0c74a3bbcfa76c1c0889b2af39ebb84f9950eb5ae5dd1d976ccd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"45888832c8fee65bfcc0a82bc8bb932e5906d8f25722fc7e89cc811f9eb0813b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"4983ace40cc218681b7857e0423a3923e3cd26d87c1638d400339c6f41432348","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"4ae69f49ae8aa5592b66ab89cbf16ca51a19caab61da63ea271f0cd5d3df03a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"4de42c0a3f9058e9fc17d4498714fc862d677cabc34be3a326710e7585ed6749","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"4e35251d59aa38391eacf4ed5d56476274f302b1101c8009ae40d2840a7b020e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"4ffa0e21d93f5b8744bf5d71747ced5f30b82b6965f0f26a528a4f3fc0cdcf44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"570ed92628f58ce24785fabbda78656146af9560440d48ec970e8f14e5ee28d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"58a7665130289bc8bf8b38124a9d7256cf3067a08ec04ee7abba242447a0f9ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"5e9c061b227b9a0c906c7efbd9762f8060022c94e57582895b25b5bfb6464c8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"710f2c56702d12963db949df115d707494485a0cd1fc368fc7b342aa1d4243c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"712565660509c471aac497011b6194673b46130351ac7c2426ddd7f50f09b079","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"80b27ff2326f3e16cba0084498452dbd599006a6d2a03d2e27d13740f31cc614","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"9923ba6ff4c141da7c0325a9780e2b9ee671cb6a111f5dd3b6a6fae3895c835a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"9cc879374e15052498fe107f632241d60262d11e582f3cd75ba610c34972a164","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"a0944cbb3759f21fd5aca543eb03916918011ad34b27ffa59dc9a6ce1fec0a02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"a42ddb62e29cf0025c5bf982292f3974b36a5cbe56eeef9956e3eda26efc8d22","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"a4add8c8ab59e6141afc7b6e22addd6d022fc04df75730792951cdb459e0490c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"a6003df04dcefb46b248b60ceda04a32a6dcb374cd38b2b36a2dfd3d0e9d3a92","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"ac777b95864ccc727bf73b81fba7efcbcb834d99c7a83a5d67206444f047b209","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"ae4f2d93567f4d177b92e19ba66d3be7de5e37d61f60b2ac11a00bcc5ae24e63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"aec8701d23db866ae4cea382008cc5c9a79ecf31566fd5fd40a80d0539d11818","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"b306455abc970335937504a0b8dbf970342a6cddaf65ead1d47c1cd26ac8e503","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"b332eacdf6ea48b33f57793593fd00c2a7f0e83eb6ab4135ca119dcff4706dbc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"b7a5492787b566bbf536961e0da3c15e740ed4aa66ba50105f398fb23abee884","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"baf99862696f1ceabf3420f805862dbaaabef70f1ef0346cf338ca1a715659bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"c7fb8276f580095e6f8ef823c7dde1335bc3adadbbd1dc163b3f43085da14b7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"d335cd7d56763119aaf2ae0bfa63da4b818d848de6700bc4e65e49a7639fd405","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"d98b2626efa4fca035fec43c17b508a41d2c22542d7c5d100f4e7bc76cfe9680","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"defba06caca5b48f794a13c44769520226a187e375516a67a01a20d96e65c3c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"dfc17cec067ab4b2e66735b77d4c61f6dc5374c99f12a2e2ae80dd113cf13e37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"e98c722279619b6f44c7448b6d51b5e98d3de275d4306b57bec8a68c891b8553","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"ed2445a883ec43a5f79308aae46436db53967da9b3d7e4549128c5dc72d3a5ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"ee2c2563fe930c718c0fc4b848650ad0233fc9148a41a1d6257d79e9e7c79e30","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"efe21dc4f1633db367f4a633d5271810146f574c23abad6499cf2bb043236415","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"f07f590c5a8560ff8119cd24252b714d16c80267fe27a479f40dd4fb4cf17c16","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"f81fa22284e45a6764e5463480983eb8429545ebc5e73f81fde8cd24f7e51e5c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"fb68c0506afca9dbb7157bf21fe830abf97281976b1bd452bf568279ca022ce2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"fe3d89e47ac90ac143320e51c687bc46c5041419622db40d299e6bbba749611e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"67e9146bba6a263169b2663e0ece2eff489af41d3e74f4e915f7ce1b74288068","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":5,"runs_affected":3,"failed_runs":3,"run_impact_percent":33.333333333333336,"anchor_occurrences":5,"after_last_push_count":12,"trend_counts":[0,1,0,0,0,0,5],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-20","2026-08-03"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-08T02:00:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d8e6c0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-rkc5mbl4jhhf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/56d1eae1-1c5b-4a4d-be74-e342e7fb8f62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/56d1eae1-1c5b-4a4d-be74-e342e7fb8f62\",\n      \"name\": \"56d1eae1-1c5b-4a4d-be74-e342e7fb8f62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T22:51:09.443199677Z\",\n      \"endTime\": \"2026-08-13T23:10:11.643927Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d8e6c0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-rkc5mbl4jhhf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/56d1eae1-1c5b-4a4d-be74-e342e7fb8f62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/56d1eae1-1c5b-4a4d-be74-e342e7fb8f62\",\n      \"name\": \"56d1eae1-1c5b-4a4d-be74-e342e7fb8f62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T22:51:09.443199677Z\",\n      \"endTime\": \"2026-08-13T23:10:11.643927Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110a340\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-l22tf5tl8zxs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/d3ea7256-0b78-469e-a13e-56a00e17bf30\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/d3ea7256-0b78-469e-a13e-56a00e17bf30\",\n      \"name\": \"d3ea7256-0b78-469e-a13e-56a00e17bf30\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T22:13:33.517913078Z\",\n      \"endTime\": \"2026-08-13T22:32:39.987880545Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110a340\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-l22tf5tl8zxs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/d3ea7256-0b78-469e-a13e-56a00e17bf30\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/d3ea7256-0b78-469e-a13e-56a00e17bf30\",\n      \"name\": \"d3ea7256-0b78-469e-a13e-56a00e17bf30\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T22:13:33.517913078Z\",\n      \"endTime\": \"2026-08-13T22:32:39.987880545Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-20-rnm2tvm2mrmp/cluster-candidate-4-20-qwtx8f should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c08b00\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-qwtx8f: failed waiting for cluster=\"cluster-candidate-4-20-qwtx8f\" in resourcegroup=\"rg-candidate-4-20-rnm2tvm2mrmp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91167263-7a29-4fa4-99d6-8166e3b4d43c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91167263-7a29-4fa4-99d6-8166e3b4d43c\",\n      \"name\": \"91167263-7a29-4fa4-99d6-8166e3b4d43c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T17:30:23.962927203Z\",\n      \"endTime\": \"2026-08-13T17:49:28.261598664Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-20-rnm2tvm2mrmp/cluster-candidate-4-20-qwtx8f should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c08b00\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-qwtx8f: failed waiting for cluster=\"cluster-candidate-4-20-qwtx8f\" in resourcegroup=\"rg-candidate-4-20-rnm2tvm2mrmp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91167263-7a29-4fa4-99d6-8166e3b4d43c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91167263-7a29-4fa4-99d6-8166e3b4d43c\",\n      \"name\": \"91167263-7a29-4fa4-99d6-8166e3b4d43c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T17:30:23.962927203Z\",\n      \"endTime\": \"2026-08-13T17:49:28.261598664Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2088019536656732160","occurred_at":"2026-08-13T21:47:05Z","signature_id":"9f090ed1e62064ca490c207349dc600d3394d354f5c7e415efb839f8bb9c1c80","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2088019536656732160","occurred_at":"2026-08-13T21:47:05Z","signature_id":"a2e879187cb6972854c4075c1f2c445ce736f5032aa71780d6b866151a01a179","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","signature_id":"4e90b9d998f42658ba6457f2b76799a9f7e58cce249d22bbff156a35e6438459","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","signature_id":"438feed7d6aa6c7b27f44db5135ec2496eb739806679ce21b0a68d1334a6abc8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","signature_id":"53ea9738f993f6a47e2afe9b16549b79000eb355a3b59cb715117ac5adabcd88","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":44.44444444444444,"anchor_occurrences":4,"after_last_push_count":29,"trend_counts":[0,0,0,0,0,4,4],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-20","2026-07-27","2026-08-03"],"prior_runs_affected":29,"prior_last_seen_at":"2026-08-05T23:00:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","signature_id":"9f6f42bed68e6d3b86d81931d02f06532f74fc43be808bc0172db5fcca80bec4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"d712d5ce17264f0ef550496665e2a7bd7d67dfac479ff00deed5bcb5e610f7c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087827320206266368","occurred_at":"2026-08-13T09:03:17Z","signature_id":"d712d5ce17264f0ef550496665e2a7bd7d67dfac479ff00deed5bcb5e610f7c9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":22.22222222222222,"anchor_occurrences":4,"after_last_push_count":128,"trend_counts":[0,0,0,0,0,5,4],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-03"],"prior_runs_affected":22,"prior_last_seen_at":"2026-08-06T12:27:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-4-22-gc96gfnnfh7f/cluster-nightly-4-22-hjgzzn should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d44860\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-hjgzzn: failed waiting for cluster=\"cluster-nightly-4-22-hjgzzn\" in resourcegroup=\"rg-nightly-4-22-gc96gfnnfh7f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b8781edb-7125-44c1-89c9-ab2300095ebb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b8781edb-7125-44c1-89c9-ab2300095ebb\",\n      \"name\": \"b8781edb-7125-44c1-89c9-ab2300095ebb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T23:32:10.892570447Z\",\n      \"endTime\": \"2026-08-13T23:51:11.867428218Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-4-22-gc96gfnnfh7f/cluster-nightly-4-22-hjgzzn should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d44860\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-hjgzzn: failed waiting for cluster=\"cluster-nightly-4-22-hjgzzn\" in resourcegroup=\"rg-nightly-4-22-gc96gfnnfh7f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b8781edb-7125-44c1-89c9-ab2300095ebb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b8781edb-7125-44c1-89c9-ab2300095ebb\",\n      \"name\": \"b8781edb-7125-44c1-89c9-ab2300095ebb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T23:32:10.892570447Z\",\n      \"endTime\": \"2026-08-13T23:51:11.867428218Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000880340\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-8r9tpgrklzkp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/07101123-d638-438a-a217-7468c9aedf3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/07101123-d638-438a-a217-7468c9aedf3f\",\n      \"name\": \"07101123-d638-438a-a217-7468c9aedf3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T16:56:50.524067582Z\",\n      \"endTime\": \"2026-08-13T17:15:56.988850244Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000880340\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-8r9tpgrklzkp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/07101123-d638-438a-a217-7468c9aedf3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/07101123-d638-438a-a217-7468c9aedf3f\",\n      \"name\": \"07101123-d638-438a-a217-7468c9aedf3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T16:56:50.524067582Z\",\n      \"endTime\": \"2026-08-13T17:15:56.988850244Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b94420\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-qbkwwv7bvchw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3e0badea-7de1-4906-94f7-05bdf32932c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3e0badea-7de1-4906-94f7-05bdf32932c7\",\n      \"name\": \"3e0badea-7de1-4906-94f7-05bdf32932c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T16:57:00.949272396Z\",\n      \"endTime\": \"2026-08-13T17:16:06.930089665Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b94420\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-qbkwwv7bvchw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3e0badea-7de1-4906-94f7-05bdf32932c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3e0badea-7de1-4906-94f7-05bdf32932c7\",\n      \"name\": \"3e0badea-7de1-4906-94f7-05bdf32932c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T16:57:00.949272396Z\",\n      \"endTime\": \"2026-08-13T17:16:06.930089665Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2088038044278984704","occurred_at":"2026-08-13T23:00:38Z","signature_id":"a6dc99ecaaca936b7dec45c6efa92ed7510efcec862beaed09ca307257ca6804","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","signature_id":"1e8a5190c6b519e6bcdb5cd13427d10302182e3754b2ed5028646e0a7266d796","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","signature_id":"1f8d2eaa98ff25669a7eeb95b723a4de8d2fa8764a770525baafc46df1a53cd9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","signature_id":"4662f084d88a96bedec02db1e5cc94bd0b5ab04413acfc35afcac56c685d173e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":22.22222222222222,"anchor_occurrences":4,"after_last_push_count":30,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-13","2026-07-20","2026-07-27","2026-08-03"],"prior_runs_affected":26,"prior_last_seen_at":"2026-08-06T23:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:759]: failed to create nodepool npdg-421-419\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011f8a20\u003e: \n    failed to create NodePool npdg-421-419: failed waiting for nodepool=\"npdg-421-419\" for cluster \"np-dg-minor-pr5df2\" in resourcegroup=\"rg-np-dg-minor-pr5df2-4xk5xp2xhfm2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/879ca928-cb37-4db6-81e7-46a882044a50\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/879ca928-cb37-4db6-81e7-46a882044a50\",\n      \"name\": \"879ca928-cb37-4db6-81e7-46a882044a50\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T13:16:32.109830139Z\",\n      \"endTime\": \"2026-08-13T13:35:38.839757933Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:759]: failed to create nodepool npdg-421-419\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011f8a20\u003e: \n    failed to create NodePool npdg-421-419: failed waiting for nodepool=\"npdg-421-419\" for cluster \"np-dg-minor-pr5df2\" in resourcegroup=\"rg-np-dg-minor-pr5df2-4xk5xp2xhfm2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/879ca928-cb37-4db6-81e7-46a882044a50\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/879ca928-cb37-4db6-81e7-46a882044a50\",\n      \"name\": \"879ca928-cb37-4db6-81e7-46a882044a50\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T13:16:32.109830139Z\",\n      \"endTime\": \"2026-08-13T13:35:38.839757933Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f60480\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-x78c7dcw45gl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f775432f-7cac-483a-af3d-cdc0bc7c5113\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f775432f-7cac-483a-af3d-cdc0bc7c5113\",\n      \"name\": \"f775432f-7cac-483a-af3d-cdc0bc7c5113\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T12:57:19.306229714Z\",\n      \"endTime\": \"2026-08-13T13:16:27.918607796Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f60480\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-x78c7dcw45gl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f775432f-7cac-483a-af3d-cdc0bc7c5113\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f775432f-7cac-483a-af3d-cdc0bc7c5113\",\n      \"name\": \"f775432f-7cac-483a-af3d-cdc0bc7c5113\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T12:57:19.306229714Z\",\n      \"endTime\": \"2026-08-13T13:16:27.918607796Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:152]: failed to create node pool npupgrade-4-20 with version 4.20.32\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e91280\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-4f46mm\" in resourcegroup=\"rg-np-version-upgrade-4f46mm-wzwtwbsl9hk4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/29565f62-084a-40db-9076-e7ac51f80f6d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/29565f62-084a-40db-9076-e7ac51f80f6d\",\n      \"name\": \"29565f62-084a-40db-9076-e7ac51f80f6d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T10:53:13.351872344Z\",\n      \"endTime\": \"2026-08-13T11:12:13.375278226Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:152]: failed to create node pool npupgrade-4-20 with version 4.20.32\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e91280\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-4f46mm\" in resourcegroup=\"rg-np-version-upgrade-4f46mm-wzwtwbsl9hk4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/29565f62-084a-40db-9076-e7ac51f80f6d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/29565f62-084a-40db-9076-e7ac51f80f6d\",\n      \"name\": \"29565f62-084a-40db-9076-e7ac51f80f6d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T10:53:13.351872344Z\",\n      \"endTime\": \"2026-08-13T11:12:13.375278226Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","signature_id":"2fc25500d7bff95fdcd99958c0b17bb722c42af6f861b04da0d5491067520d71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","signature_id":"de9fba52a960b5d509c226c1c083c6d0b28085e513bcaecc67e116c6984c9df4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087827320206266368","occurred_at":"2026-08-13T09:03:17Z","signature_id":"7896e80f3d43b9ff9e34ed42eb43701faf6b382200f9cda7c5ea503730f39acb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087827320206266368","occurred_at":"2026-08-13T09:03:17Z","signature_id":"8bd705a5eb53385882f688bf9ed38b343f1f635530b9a0d3639f8756cd03121b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"0438873facb9dd98e74695c849423f31b775588ec5b0ef7ead10a8182c141d31","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":22.22222222222222,"anchor_occurrences":3,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,1,3],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-03"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-06T12:27:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-vzgnsfg29j2j/cluster-candidate-4-22-5x9pt4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008be700\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-5x9pt4: failed waiting for cluster=\"cluster-candidate-4-22-5x9pt4\" in resourcegroup=\"rg-candidate-4-22-vzgnsfg29j2j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8131ef38-f0b6-49e5-94b9-386607403e5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8131ef38-f0b6-49e5-94b9-386607403e5f\",\n      \"name\": \"8131ef38-f0b6-49e5-94b9-386607403e5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T17:31:43.935377005Z\",\n      \"endTime\": \"2026-08-13T17:50:48.106242949Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-vzgnsfg29j2j/cluster-candidate-4-22-5x9pt4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008be700\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-5x9pt4: failed waiting for cluster=\"cluster-candidate-4-22-5x9pt4\" in resourcegroup=\"rg-candidate-4-22-vzgnsfg29j2j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8131ef38-f0b6-49e5-94b9-386607403e5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8131ef38-f0b6-49e5-94b9-386607403e5f\",\n      \"name\": \"8131ef38-f0b6-49e5-94b9-386607403e5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T17:31:43.935377005Z\",\n      \"endTime\": \"2026-08-13T17:50:48.106242949Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000540280\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-5mdmhgbzh58p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/af8ee527-2640-4dcb-92f9-765f4b0064ce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/af8ee527-2640-4dcb-92f9-765f4b0064ce\",\n      \"name\": \"af8ee527-2640-4dcb-92f9-765f4b0064ce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T16:56:53.856645111Z\",\n      \"endTime\": \"2026-08-13T17:15:56.849316189Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000540280\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-5mdmhgbzh58p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/af8ee527-2640-4dcb-92f9-765f4b0064ce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/af8ee527-2640-4dcb-92f9-765f4b0064ce\",\n      \"name\": \"af8ee527-2640-4dcb-92f9-765f4b0064ce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T16:56:53.856645111Z\",\n      \"endTime\": \"2026-08-13T17:15:56.849316189Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e84520\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-drrx6cd6559n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b56ccafd-4383-4ad9-9f24-2989a66e180f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b56ccafd-4383-4ad9-9f24-2989a66e180f\",\n      \"name\": \"b56ccafd-4383-4ad9-9f24-2989a66e180f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T11:56:40.932047182Z\",\n      \"endTime\": \"2026-08-13T12:15:46.19337299Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e84520\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-drrx6cd6559n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b56ccafd-4383-4ad9-9f24-2989a66e180f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b56ccafd-4383-4ad9-9f24-2989a66e180f\",\n      \"name\": \"b56ccafd-4383-4ad9-9f24-2989a66e180f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T11:56:40.932047182Z\",\n      \"endTime\": \"2026-08-13T12:15:46.19337299Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","signature_id":"3470f6e6551778c521fae7611d06158bd7b0f2d9f0f18172ea64158c3fb3b4bd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","signature_id":"5ef680a9f58e5968f085acafa4181a8cca9c772497009d797b9cbc55216675fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","signature_id":"5b50bdacf46db4a62ac301c86303aa6cdc04641a80f7a89024a59e1b84faa061","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"cb8ca43a79863985fe5fa47ef1bbe80d7bc25200d089461928007bb1c0bcd82f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":22.22222222222222,"anchor_occurrences":3,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,3,3],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-mgrpt82ln4bh/cluster-candidate-4-21-cvgp9r should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010de6c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-cvgp9r: failed waiting for cluster=\"cluster-candidate-4-21-cvgp9r\" in resourcegroup=\"rg-candidate-4-21-mgrpt82ln4bh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/c11e3bda-7b18-4cf4-9d5e-bc8c82684e7b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/c11e3bda-7b18-4cf4-9d5e-bc8c82684e7b\",\n      \"name\": \"c11e3bda-7b18-4cf4-9d5e-bc8c82684e7b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T17:30:22.500953939Z\",\n      \"endTime\": \"2026-08-13T17:49:28.209822365Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-mgrpt82ln4bh/cluster-candidate-4-21-cvgp9r should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010de6c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-cvgp9r: failed waiting for cluster=\"cluster-candidate-4-21-cvgp9r\" in resourcegroup=\"rg-candidate-4-21-mgrpt82ln4bh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/c11e3bda-7b18-4cf4-9d5e-bc8c82684e7b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/c11e3bda-7b18-4cf4-9d5e-bc8c82684e7b\",\n      \"name\": \"c11e3bda-7b18-4cf4-9d5e-bc8c82684e7b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T17:30:22.500953939Z\",\n      \"endTime\": \"2026-08-13T17:49:28.209822365Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001027020\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-s6pxrqlhs4fd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/363acc2d-8995-4b0b-8042-e788a23c46c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/363acc2d-8995-4b0b-8042-e788a23c46c8\",\n      \"name\": \"363acc2d-8995-4b0b-8042-e788a23c46c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T16:56:49.931217747Z\",\n      \"endTime\": \"2026-08-13T17:15:56.947670699Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001027020\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-s6pxrqlhs4fd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/363acc2d-8995-4b0b-8042-e788a23c46c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/363acc2d-8995-4b0b-8042-e788a23c46c8\",\n      \"name\": \"363acc2d-8995-4b0b-8042-e788a23c46c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T16:56:49.931217747Z\",\n      \"endTime\": \"2026-08-13T17:15:56.947670699Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-mm6p45mrvjqv/cluster-candidate-5-0-7ql8bc should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e01820\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-7ql8bc: failed waiting for cluster=\"cluster-candidate-5-0-7ql8bc\" in resourcegroup=\"rg-candidate-5-0-mm6p45mrvjqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/34962d53-53be-48ee-b96e-e2fa47155f69\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/34962d53-53be-48ee-b96e-e2fa47155f69\",\n      \"name\": \"34962d53-53be-48ee-b96e-e2fa47155f69\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T12:30:12.239477716Z\",\n      \"endTime\": \"2026-08-13T12:49:17.224447305Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-mm6p45mrvjqv/cluster-candidate-5-0-7ql8bc should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e01820\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-7ql8bc: failed waiting for cluster=\"cluster-candidate-5-0-7ql8bc\" in resourcegroup=\"rg-candidate-5-0-mm6p45mrvjqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/34962d53-53be-48ee-b96e-e2fa47155f69\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/34962d53-53be-48ee-b96e-e2fa47155f69\",\n      \"name\": \"34962d53-53be-48ee-b96e-e2fa47155f69\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T12:30:12.239477716Z\",\n      \"endTime\": \"2026-08-13T12:49:17.224447305Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","signature_id":"50350edc4c7d5f835e6689a0d4e14af05941274d2afe9dfd3f5ccaac276423bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","signature_id":"ba7494a6c8814df7c2bba6f4ada3833c130e86360b32c05f3ce3226d149c0de1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","signature_id":"4ea026269c6c2776cf2d008e8e33582e8c4d04ad5a222a3c4eda1afa2cacb4af","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-1-jxrdzr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jxrdzr in resource group rg-nightly-5-1-ps74xj8djqv9, error: failed waiting for cluster=\"cluster-nightly-5-1-jxrdzr\" in resourcegroup=\"rg-nightly-5-1-ps74xj8djqv9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jxrdzr in resource group rg-nightly-5-1-ps74xj8djqv9, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":22.22222222222222,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-13","2026-07-20","2026-07-27","2026-08-03"],"prior_runs_affected":36,"prior_last_seen_at":"2026-08-09T23:00:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-ps74xj8djqv9/cluster-nightly-5-1-jxrdzr should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0018b6570\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-jxrdzr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jxrdzr in resource group rg-nightly-5-1-ps74xj8djqv9, error: failed waiting for cluster=\"cluster-nightly-5-1-jxrdzr\" in resourcegroup=\"rg-nightly-5-1-ps74xj8djqv9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jxrdzr in resource group rg-nightly-5-1-ps74xj8djqv9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-ps74xj8djqv9/cluster-nightly-5-1-jxrdzr should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0018b6570\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-jxrdzr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jxrdzr in resource group rg-nightly-5-1-ps74xj8djqv9, error: failed waiting for cluster=\"cluster-nightly-5-1-jxrdzr\" in resourcegroup=\"rg-nightly-5-1-ps74xj8djqv9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jxrdzr in resource group rg-nightly-5-1-ps74xj8djqv9, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001489350\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-z2tkbzk2p5xk, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-z2tkbzk2p5xk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-z2tkbzk2p5xk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001489350\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-z2tkbzk2p5xk, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-z2tkbzk2p5xk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-z2tkbzk2p5xk, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:151]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016d02a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-zksnvzwlwp47\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-zksnvzwlwp47, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:151]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016d02a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-zksnvzwlwp47\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-zksnvzwlwp47, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2088038044278984704","occurred_at":"2026-08-13T23:00:38Z","signature_id":"50d20647dc4e343420ed4311b1653506de8b52adcdc8d7e9904de0aebda52739","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2088038044278984704","occurred_at":"2026-08-13T23:00:38Z","signature_id":"7c048170bd9fa51a7ee93f4d46bf9fc646bd0775fe746ada1f6fef606b9ae017","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"75c093f2bd0988ddc22af9e2f1728a3622a04d8d837f81fc566043f24916c7e1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":22.22222222222222,"anchor_occurrences":2,"after_last_push_count":9,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-20","2026-07-27"],"prior_runs_affected":9,"prior_last_seen_at":"2026-08-02T09:15:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa4e3f3fddf55a14f372dba46754c68c0e6985edc043acb16ad92a33f436f738","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:314]: failed to create HCP cluster np-noedge-q8wb29\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e90b40\u003e: \n    failed to create HCP cluster np-noedge-q8wb29: failed waiting for cluster=\"np-noedge-q8wb29\" in resourcegroup=\"rg-np-noedge-q8wb29-scqh47gqfgtk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/49e390d3-8264-4961-b591-2ba8a2528384\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/49e390d3-8264-4961-b591-2ba8a2528384\",\n      \"name\": \"49e390d3-8264-4961-b591-2ba8a2528384\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T18:04:01.864548732Z\",\n      \"endTime\": \"2026-08-13T18:23:09.31047489Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:314]: failed to create HCP cluster np-noedge-q8wb29\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e90b40\u003e: \n    failed to create HCP cluster np-noedge-q8wb29: failed waiting for cluster=\"np-noedge-q8wb29\" in resourcegroup=\"rg-np-noedge-q8wb29-scqh47gqfgtk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/49e390d3-8264-4961-b591-2ba8a2528384\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/49e390d3-8264-4961-b591-2ba8a2528384\",\n      \"name\": \"49e390d3-8264-4961-b591-2ba8a2528384\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T18:04:01.864548732Z\",\n      \"endTime\": \"2026-08-13T18:23:09.31047489Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","signature_id":"e36c1f212cf181bdf7c7a3efef0227f22922a029910af135e4fa995832869820","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a21c55609829272b5f173dd8ce8e25d8c4d9a8eec84afefcdb262ef97c91acb7","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a8f660\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-x6t6kwx6bp42\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/65d119df-660f-4e58-9993-5f6118c5be2d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/65d119df-660f-4e58-9993-5f6118c5be2d\",\n      \"name\": \"65d119df-660f-4e58-9993-5f6118c5be2d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T16:57:36.200799801Z\",\n      \"endTime\": \"2026-08-13T17:16:36.805943703Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a8f660\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-x6t6kwx6bp42\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/65d119df-660f-4e58-9993-5f6118c5be2d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/65d119df-660f-4e58-9993-5f6118c5be2d\",\n      \"name\": \"65d119df-660f-4e58-9993-5f6118c5be2d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T16:57:36.200799801Z\",\n      \"endTime\": \"2026-08-13T17:16:36.805943703Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","signature_id":"66ef78e183930b4044539e132fae6fd77f008b6b1e5c0e9142bb5f2c6f1f3265","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d9509a4930132704b80afcf0f9ab5f4573c4e906011498304e9125ac11a4a978","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-fl5dzt9shx72/cluster-candidate-5-0-fsc67s should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000644380\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-fsc67s: failed waiting for cluster=\"cluster-candidate-5-0-fsc67s\" in resourcegroup=\"rg-candidate-5-0-fl5dzt9shx72\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f29b4185-b022-4212-8a83-448e94357420\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f29b4185-b022-4212-8a83-448e94357420\",\n      \"name\": \"f29b4185-b022-4212-8a83-448e94357420\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T17:31:47.763208934Z\",\n      \"endTime\": \"2026-08-13T17:50:48.076190757Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-fl5dzt9shx72/cluster-candidate-5-0-fsc67s should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000644380\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-fsc67s: failed waiting for cluster=\"cluster-candidate-5-0-fsc67s\" in resourcegroup=\"rg-candidate-5-0-fl5dzt9shx72\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f29b4185-b022-4212-8a83-448e94357420\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f29b4185-b022-4212-8a83-448e94357420\",\n      \"name\": \"f29b4185-b022-4212-8a83-448e94357420\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T17:31:47.763208934Z\",\n      \"endTime\": \"2026-08-13T17:50:48.076190757Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","signature_id":"fae07ddb718bf98f6d3d1edced5cb60ae56e409927f36cedc96217c0cd7a290c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"18e7e3bddd3f5a1d71bd0a4e399cd7bd2e6186d31ff0833685a14cba11311d1d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001362c80\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-m4w85gmwnx2g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b4037bc6-8c3f-4270-affb-68a1af331f3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b4037bc6-8c3f-4270-affb-68a1af331f3f\",\n      \"name\": \"b4037bc6-8c3f-4270-affb-68a1af331f3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T16:58:24.22544747Z\",\n      \"endTime\": \"2026-08-13T17:17:27.035958879Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, packageserver, cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, olm-operator, openshift-controller-manager, catalog-operator, cluster-storage-operator, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001362c80\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-m4w85gmwnx2g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b4037bc6-8c3f-4270-affb-68a1af331f3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b4037bc6-8c3f-4270-affb-68a1af331f3f\",\n      \"name\": \"b4037bc6-8c3f-4270-affb-68a1af331f3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T16:58:24.22544747Z\",\n      \"endTime\": \"2026-08-13T17:17:27.035958879Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, packageserver, cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, olm-operator, openshift-controller-manager, catalog-operator, cluster-storage-operator, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","signature_id":"e815ffbc8459f58bfe52e56bcd82851f27a02fcad4d28992925670cbef6453f5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"aeb99d9b1ad881fcb99495f400b224fcad0bc6f5c9ff6c3ffad9218a1eb1ffa7","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011dc440\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-z7twstkk877q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/ae54c7de-b8d4-4041-9176-22b9c96c29a5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/ae54c7de-b8d4-4041-9176-22b9c96c29a5\",\n      \"name\": \"ae54c7de-b8d4-4041-9176-22b9c96c29a5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T16:58:22.806879081Z\",\n      \"endTime\": \"2026-08-13T17:17:27.029777623Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.y5u5u9r0s4y5z1m.bj4u.westeurope.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011dc440\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-z7twstkk877q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/ae54c7de-b8d4-4041-9176-22b9c96c29a5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/ae54c7de-b8d4-4041-9176-22b9c96c29a5\",\n      \"name\": \"ae54c7de-b8d4-4041-9176-22b9c96c29a5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T16:58:22.806879081Z\",\n      \"endTime\": \"2026-08-13T17:17:27.029777623Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.y5u5u9r0s4y5z1m.bj4u.westeurope.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","signature_id":"a9e21576b8c2932aef14ea318a2a302475569455a7e3c670e4e4b6c84c04d9f2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79d42a294edfd7eaa19516eb5e39cb2fbbc57c83345983aea3a63ac716c4f9fa","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002d209e0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-5mmdptfkb4qp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/024da1c7-6df0-487c-8a47-bb2fc6fb63cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/024da1c7-6df0-487c-8a47-bb2fc6fb63cc\",\n      \"name\": \"024da1c7-6df0-487c-8a47-bb2fc6fb63cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T11:57:40.70442981Z\",\n      \"endTime\": \"2026-08-13T12:16:46.395053793Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002d209e0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-5mmdptfkb4qp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/024da1c7-6df0-487c-8a47-bb2fc6fb63cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/024da1c7-6df0-487c-8a47-bb2fc6fb63cc\",\n      \"name\": \"024da1c7-6df0-487c-8a47-bb2fc6fb63cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T11:57:40.70442981Z\",\n      \"endTime\": \"2026-08-13T12:16:46.395053793Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","signature_id":"0421de590218865da65b3af252c9dfa3cd7c3f55271342d96e5089c485c514a1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"46ba20bd0022bbab2077ca36c67330fccd8ca0996c8be8400f5fdb8a593b354d","failure_pattern":"VerifyVMOSDiskCustomerEncryption(nodePool=\u003cnodepool\u003e) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-zpk2h-89fng_OSDisk for VM des-encrypt-des-np-zpk2h-89fng has encryption type EncryptionAtRestWithPlatformKey, expected","search_query":"VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-zpk2h-89fng_OSDisk for VM des-encrypt-des-np-zpk2h-89fng has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:163]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc002918a50\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-zpk2h-89fng_OSDisk for VM des-encrypt-des-np-zpk2h-89fng has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    disk des-encrypt-des-np-zpk2h-f5zfj_OSDisk for VM des-encrypt-des-np-zpk2h-f5zfj has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:163]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc002918a50\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-zpk2h-89fng_OSDisk for VM des-encrypt-des-np-zpk2h-89fng has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    disk des-encrypt-des-np-zpk2h-f5zfj_OSDisk for VM des-encrypt-des-np-zpk2h-f5zfj has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2088038044278984704","occurred_at":"2026-08-13T23:00:38Z","signature_id":"6b8fcd2703f57320f816bd4e0c92f70acab07d54b540ea5a0f114864c8d8bc88","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-xtxbpdvbqtqq\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-13"],"prior_runs_affected":6,"prior_last_seen_at":"2026-07-19T02:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:197]: failed to create node pool version 4.19.7 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00131aa00\u003e: \n    failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-xtxbpdvbqtqq\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:197]: failed to create node pool version 4.19.7 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00131aa00\u003e: \n    failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-xtxbpdvbqtqq\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","signature_id":"71d707d8d230945319eacc89a89423b1ed0d8831a6f5ba1ad76723820292b2d8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":12,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-08-07..2026-08-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-13","2026-07-20","2026-07-27","2026-08-03"],"prior_runs_affected":13,"prior_last_seen_at":"2026-08-05T12:11:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n    \u003c*errors.joinError | 0xc0027b4a98\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-fzfll.apps.aro.d9s7k9e5c7c4b4u.01ef.westeurope.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n    \u003c*errors.joinError | 0xc0027b4a98\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-fzfll.apps.aro.d9s7k9e5c7c4b4u.01ef.westeurope.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","signature_id":"32533af6ce35e8298dd5671118e3af98dc4c24311c09f33c4046c07eb485d35b","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}