{"meta":{"start_date":"2026-08-14","end_date":"2026-08-14","start_at":"2026-08-14T00:00:00Z","end_at":"2026-08-15T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T16:53:10Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":3,"failed_runs":3,"raw_occurrences":17,"matched_occurrences":17,"runs_affected":3},"rows":[{"environment":"prod","failure_pattern_id":"0438873facb9dd98e74695c849423f31b775588ec5b0ef7ead10a8182c141d31","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":5,"after_last_push_count":11,"trend_counts":[0,0,0,0,1,3,5],"trend_range":"2026-08-08..2026-08-14 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-03"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-06T12:27:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-gw8xvm5qcvv2/cluster-candidate-4-21-pp5hmz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e194a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-pp5hmz: failed waiting for cluster=\"cluster-candidate-4-21-pp5hmz\" in resourcegroup=\"rg-candidate-4-21-gw8xvm5qcvv2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/84898c46-00b7-4dd1-a53c-b2ca17b692d9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/84898c46-00b7-4dd1-a53c-b2ca17b692d9\",\n      \"name\": \"84898c46-00b7-4dd1-a53c-b2ca17b692d9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-14T03:16:14.161226952Z\",\n      \"endTime\": \"2026-08-14T03:35:22.643256613Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-gw8xvm5qcvv2/cluster-candidate-4-21-pp5hmz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e194a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-pp5hmz: failed waiting for cluster=\"cluster-candidate-4-21-pp5hmz\" in resourcegroup=\"rg-candidate-4-21-gw8xvm5qcvv2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/84898c46-00b7-4dd1-a53c-b2ca17b692d9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/84898c46-00b7-4dd1-a53c-b2ca17b692d9\",\n      \"name\": \"84898c46-00b7-4dd1-a53c-b2ca17b692d9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-14T03:16:14.161226952Z\",\n      \"endTime\": \"2026-08-14T03:35:22.643256613Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-hndkzs2fvmfl/cluster-candidate-4-22-6t6qsl should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004aed60\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-6t6qsl: failed waiting for cluster=\"cluster-candidate-4-22-6t6qsl\" in resourcegroup=\"rg-candidate-4-22-hndkzs2fvmfl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/cc154f54-f074-4deb-840e-e31ead99c2cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/cc154f54-f074-4deb-840e-e31ead99c2cf\",\n      \"name\": \"cc154f54-f074-4deb-840e-e31ead99c2cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-14T03:17:23.637851587Z\",\n      \"endTime\": \"2026-08-14T03:36:32.688237575Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-hndkzs2fvmfl/cluster-candidate-4-22-6t6qsl should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004aed60\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-6t6qsl: failed waiting for cluster=\"cluster-candidate-4-22-6t6qsl\" in resourcegroup=\"rg-candidate-4-22-hndkzs2fvmfl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/cc154f54-f074-4deb-840e-e31ead99c2cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/cc154f54-f074-4deb-840e-e31ead99c2cf\",\n      \"name\": \"cc154f54-f074-4deb-840e-e31ead99c2cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-14T03:17:23.637851587Z\",\n      \"endTime\": \"2026-08-14T03:36:32.688237575Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00111ad20\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-xbqvklmbb74c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b02104af-2b12-4182-8206-cc4bfa7d8e2e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b02104af-2b12-4182-8206-cc4bfa7d8e2e\",\n      \"name\": \"b02104af-2b12-4182-8206-cc4bfa7d8e2e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-14T02:45:19.145147789Z\",\n      \"endTime\": \"2026-08-14T03:04:21.089268067Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00111ad20\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-xbqvklmbb74c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b02104af-2b12-4182-8206-cc4bfa7d8e2e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b02104af-2b12-4182-8206-cc4bfa7d8e2e\",\n      \"name\": \"b02104af-2b12-4182-8206-cc4bfa7d8e2e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-14T02:45:19.145147789Z\",\n      \"endTime\": \"2026-08-14T03:04:21.089268067Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2088087349807288320","occurred_at":"2026-08-14T02:16:33Z","signature_id":"0ad9936d7c8041b4fdf94af2814d621abb0cf4a8f1479b45f20f53eddf27669d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2088087349807288320","occurred_at":"2026-08-14T02:16:33Z","signature_id":"27d0bcae76793f29e7a156122adf94f68e03e6b5c7a755f777ad6d7c99e728e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2088087349807288320","occurred_at":"2026-08-14T02:16:33Z","signature_id":"4ae0910329dfc8598bf54929f030af4782b62690d98bc6a20f02bfd6f751e063","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2088087349807288320","occurred_at":"2026-08-14T02:16:33Z","signature_id":"9e6f24df29b082a3f9243e3d051f6e279b6442e6566ffd31c8c51ef416fbd822","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2088087349807288320","occurred_at":"2026-08-14T02:16:33Z","signature_id":"fc0f17fbaf9dfa72f3239db1825904251cadf2a4ae78af9a047e876e898c9823","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"cb8ca43a79863985fe5fa47ef1bbe80d7bc25200d089461928007bb1c0bcd82f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":2,"after_last_push_count":8,"trend_counts":[0,0,0,0,3,3,2],"trend_range":"2026-08-08..2026-08-14 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000885b20\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-57plvn4qvpmn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/36272d57-fc14-4249-a0c9-36ab337c304f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/36272d57-fc14-4249-a0c9-36ab337c304f\",\n      \"name\": \"36272d57-fc14-4249-a0c9-36ab337c304f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-14T02:45:02.566197044Z\",\n      \"endTime\": \"2026-08-14T03:04:11.269044628Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000885b20\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-57plvn4qvpmn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/36272d57-fc14-4249-a0c9-36ab337c304f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/36272d57-fc14-4249-a0c9-36ab337c304f\",\n      \"name\": \"36272d57-fc14-4249-a0c9-36ab337c304f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-14T02:45:02.566197044Z\",\n      \"endTime\": \"2026-08-14T03:04:11.269044628Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-9j77hrzg8ncp/cluster-candidate-5-0-zmjqkw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a93080\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-zmjqkw: failed waiting for cluster=\"cluster-candidate-5-0-zmjqkw\" in resourcegroup=\"rg-candidate-5-0-9j77hrzg8ncp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3dfd9a55-aee6-412e-8f37-2823b565dbaa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3dfd9a55-aee6-412e-8f37-2823b565dbaa\",\n      \"name\": \"3dfd9a55-aee6-412e-8f37-2823b565dbaa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-14T03:17:52.205226636Z\",\n      \"endTime\": \"2026-08-14T03:36:52.523278161Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-9j77hrzg8ncp/cluster-candidate-5-0-zmjqkw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a93080\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-zmjqkw: failed waiting for cluster=\"cluster-candidate-5-0-zmjqkw\" in resourcegroup=\"rg-candidate-5-0-9j77hrzg8ncp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3dfd9a55-aee6-412e-8f37-2823b565dbaa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3dfd9a55-aee6-412e-8f37-2823b565dbaa\",\n      \"name\": \"3dfd9a55-aee6-412e-8f37-2823b565dbaa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-14T03:17:52.205226636Z\",\n      \"endTime\": \"2026-08-14T03:36:52.523278161Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2088087349807288320","occurred_at":"2026-08-14T02:16:33Z","signature_id":"1e28de9d35515082af936645045a2ffd5454409a651dd3b9c70e693d1aa36367","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2088087349807288320","occurred_at":"2026-08-14T02:16:33Z","signature_id":"78812466531b713bf5080f4ee7307a67bfef08a07459b962448dc9a8906940dd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-1-9mdncv, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-9mdncv in resource group rg-nightly-5-1-m6trbc6hm8kc, error: failed waiting for cluster=\"cluster-nightly-5-1-9mdncv\" in resourcegroup=\"rg-nightly-5-1-m6trbc6hm8kc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-9mdncv in resource group rg-nightly-5-1-m6trbc6hm8kc, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-08..2026-08-14 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-13","2026-07-20","2026-07-27","2026-08-03"],"prior_runs_affected":37,"prior_last_seen_at":"2026-08-09T23:00:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-m6trbc6hm8kc/cluster-nightly-5-1-9mdncv should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00095f230\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-9mdncv, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-9mdncv in resource group rg-nightly-5-1-m6trbc6hm8kc, error: failed waiting for cluster=\"cluster-nightly-5-1-9mdncv\" in resourcegroup=\"rg-nightly-5-1-m6trbc6hm8kc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-9mdncv in resource group rg-nightly-5-1-m6trbc6hm8kc, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-m6trbc6hm8kc/cluster-nightly-5-1-9mdncv should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00095f230\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-9mdncv, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-9mdncv in resource group rg-nightly-5-1-m6trbc6hm8kc, error: failed waiting for cluster=\"cluster-nightly-5-1-9mdncv\" in resourcegroup=\"rg-nightly-5-1-m6trbc6hm8kc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-9mdncv in resource group rg-nightly-5-1-m6trbc6hm8kc, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00150e6f0\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-78mmb95dktph, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-78mmb95dktph\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-78mmb95dktph, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00150e6f0\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-78mmb95dktph, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-78mmb95dktph\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-78mmb95dktph, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2088400490290221056","occurred_at":"2026-08-14T23:00:52Z","signature_id":"5ceefd4c6da8d7e9c62e0c65c9f0c1ac986146d8f5bb9d35f460f424f9c76afc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2088400490290221056","occurred_at":"2026-08-14T23:00:52Z","signature_id":"69e5494696bef0b18418d31e46487667fce68234be30807405cd4735f5ef1e95","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"273145c54586e4957a8ded0fddfea5a1e731a53a330aedb7a6233b30725d7805","failure_pattern":"(wrapped at ) RemoteFailed: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"(wrapped at control_plane_desired_version_controller.go:458) RemoteFailed: Get \"https://multi.ocp.releases.ci.openshift.org/graph?arch=multi\u0026channel=nightly-4.21\u0026id=6ba7b810-9dad-11d1-80b4-00c04fd430c8\u0026version=4.20.0\": dial tcp: lookup multi.ocp.releases.ci.openshift.org on 172.30.0.10:53: read udp 10.131.210.197:35653-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-08..2026-08-14 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:89]: failed to find upgrade targets in minor 4.21.0 for install version 4.20.0\nUnexpected error:\n    \u003c*utils.LineTrackingError | 0xc000f5bda0\u003e: \n    (wrapped at control_plane_desired_version_controller.go:458) RemoteFailed: Get \"https://multi.ocp.releases.ci.openshift.org/graph?arch=multi\u0026channel=nightly-4.21\u0026id=6ba7b810-9dad-11d1-80b4-00c04fd430c8\u0026version=4.20.0\": dial tcp: lookup multi.ocp.releases.ci.openshift.org on 172.30.0.10:53: read udp 10.131.210.197:35653-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:89]: failed to find upgrade targets in minor 4.21.0 for install version 4.20.0\nUnexpected error:\n    \u003c*utils.LineTrackingError | 0xc000f5bda0\u003e: \n    (wrapped at control_plane_desired_version_controller.go:458) RemoteFailed: Get \"https://multi.ocp.releases.ci.openshift.org/graph?arch=multi\u0026channel=nightly-4.21\u0026id=6ba7b810-9dad-11d1-80b4-00c04fd430c8\u0026version=4.20.0\": dial tcp: lookup multi.ocp.releases.ci.openshift.org on 172.30.0.10:53: read udp 10.131.210.197:35653-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2088400490290221056","occurred_at":"2026-08-14T23:00:52Z","signature_id":"ed3ff52cad3db615b8168d9664cf3988d989a3a448646f1b36249ebf3a375b1c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"11901bbac371edd7e7893806336a4161e5f7260b0d52bbaf041ebcda245c4185","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-08..2026-08-14 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000db6d40\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-76q2mvkzgs8k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/6ad92bcc-e43e-416f-a5de-dced9c7ed8b5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/6ad92bcc-e43e-416f-a5de-dced9c7ed8b5\",\n      \"name\": \"6ad92bcc-e43e-416f-a5de-dced9c7ed8b5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-14T02:43:39.311083657Z\",\n      \"endTime\": \"2026-08-14T03:02:41.194409983Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000db6d40\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-76q2mvkzgs8k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/6ad92bcc-e43e-416f-a5de-dced9c7ed8b5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/6ad92bcc-e43e-416f-a5de-dced9c7ed8b5\",\n      \"name\": \"6ad92bcc-e43e-416f-a5de-dced9c7ed8b5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-14T02:43:39.311083657Z\",\n      \"endTime\": \"2026-08-14T03:02:41.194409983Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2088087349807288320","occurred_at":"2026-08-14T02:16:33Z","signature_id":"71187dc87947b62b88576749331e6070aa390a3dedb0076b74a0cd8c9f4308ff","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":129,"trend_counts":[0,0,0,0,5,4,1],"trend_range":"2026-08-08..2026-08-14 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-03"],"prior_runs_affected":23,"prior_last_seen_at":"2026-08-06T12:27:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-5rd9rv with version 4.21.28\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004056e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-5rd9rv: failed waiting for cluster=\"np-version-upgrade-cluster-5rd9rv\" in resourcegroup=\"rg-np-version-upgrade-5rd9rv-l9ghdszlwlsv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/0e09a54a-837a-4de9-b361-ff6d83501f2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/0e09a54a-837a-4de9-b361-ff6d83501f2b\",\n      \"name\": \"0e09a54a-837a-4de9-b361-ff6d83501f2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-14T03:48:47.042378322Z\",\n      \"endTime\": \"2026-08-14T04:07:53.883454309Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-5rd9rv with version 4.21.28\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004056e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-5rd9rv: failed waiting for cluster=\"np-version-upgrade-cluster-5rd9rv\" in resourcegroup=\"rg-np-version-upgrade-5rd9rv-l9ghdszlwlsv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/0e09a54a-837a-4de9-b361-ff6d83501f2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/0e09a54a-837a-4de9-b361-ff6d83501f2b\",\n      \"name\": \"0e09a54a-837a-4de9-b361-ff6d83501f2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-14T03:48:47.042378322Z\",\n      \"endTime\": \"2026-08-14T04:07:53.883454309Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2088087349807288320","occurred_at":"2026-08-14T02:16:33Z","signature_id":"65cb1e2a5493cb6db81ed7e8f3850b34c929ea922d3f10ab42653937449d97f2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"851c19e9bf8a31605833e2995d5b896a41621549ffab7b8cb3e39813049a37ec","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-08..2026-08-14 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1e300\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-tljvzmwbv4xj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/48d4732f-5b96-418a-9969-c3a53bdbb197\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/48d4732f-5b96-418a-9969-c3a53bdbb197\",\n      \"name\": \"48d4732f-5b96-418a-9969-c3a53bdbb197\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-14T02:43:51.361386579Z\",\n      \"endTime\": \"2026-08-14T03:03:00.950264593Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1e300\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-tljvzmwbv4xj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/48d4732f-5b96-418a-9969-c3a53bdbb197\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/48d4732f-5b96-418a-9969-c3a53bdbb197\",\n      \"name\": \"48d4732f-5b96-418a-9969-c3a53bdbb197\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-14T02:43:51.361386579Z\",\n      \"endTime\": \"2026-08-14T03:03:00.950264593Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2088087349807288320","occurred_at":"2026-08-14T02:16:33Z","signature_id":"494fea040e7c516b70efb652e17ed2618fb9e611981c2cce1fca7d2ff4e5e88e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"e065b3759f60a327e04f2645401c8843144374338978ce6bc40cc04da9d3a4cb","failure_pattern":"VerifyNodesReady failed: can't list nodes in the cluster: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyNodesReady failed: can't list nodes in the cluster: Get \"https://api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io:443/api/v1/nodes\": dial tcp: lookup api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io on 172.30.0.10:53: read udp 10.131.210.197:42819-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-08..2026-08-14 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:192]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002400300\u003e: \n    VerifyCiliumOperational failed: failed to get cilium namespace: Get \"https://api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io:443/api/v1/namespaces/kube-system\": dial tcp: lookup api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io on 172.30.0.10:53: read udp 10.131.210.197:42819-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io on 172.30.0.10:53: read udp 10.131.210.197:42819-\u003e172.30.0.10:53: i/o timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": dial tcp: lookup api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io on 172.30.0.10:53: read udp 10.131.210.197:42819-\u003e172.30.0.10:53: i/o timeout\n    VerifyNodesReady failed: can't list nodes in the cluster: Get \"https://api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io:443/api/v1/nodes\": dial tcp: lookup api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io on 172.30.0.10:53: read udp 10.131.210.197:42819-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:192]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002400300\u003e: \n    VerifyCiliumOperational failed: failed to get cilium namespace: Get \"https://api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io:443/api/v1/namespaces/kube-system\": dial tcp: lookup api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io on 172.30.0.10:53: read udp 10.131.210.197:42819-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io on 172.30.0.10:53: read udp 10.131.210.197:42819-\u003e172.30.0.10:53: i/o timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": dial tcp: lookup api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io on 172.30.0.10:53: read udp 10.131.210.197:42819-\u003e172.30.0.10:53: i/o timeout\n    VerifyNodesReady failed: can't list nodes in the cluster: Get \"https://api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io:443/api/v1/nodes\": dial tcp: lookup api.cilium-cluster.f2fj.uksouth.aroapp-hcp.io on 172.30.0.10:53: read udp 10.131.210.197:42819-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2088400490290221056","occurred_at":"2026-08-14T23:00:52Z","signature_id":"ae5e18204cb9921e1b6e793e41bfd391bc2d7a551708f9a07f58367de5edac17","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9bb6208ec573a0b8c3b7a2481a968c15c62249a96577a181bc20b24ccfd7bb2e","failure_pattern":"VerifyVMOSDiskCustomerEncryption(nodePool=\u003cnodepool\u003e) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-gs4jx-fkqfk_OSDisk for VM des-encrypt-des-np-gs4jx-fkqfk has encryption type EncryptionAtRestWithPlatformKey, expected","search_query":"VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-gs4jx-fkqfk_OSDisk for VM des-encrypt-des-np-gs4jx-fkqfk has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-08..2026-08-14 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:163]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc0010db638\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-gs4jx-fkqfk_OSDisk for VM des-encrypt-des-np-gs4jx-fkqfk has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    disk des-encrypt-des-np-gs4jx-stqv5_OSDisk for VM des-encrypt-des-np-gs4jx-stqv5 has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:163]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc0010db638\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-gs4jx-fkqfk_OSDisk for VM des-encrypt-des-np-gs4jx-fkqfk has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    disk des-encrypt-des-np-gs4jx-stqv5_OSDisk for VM des-encrypt-des-np-gs4jx-stqv5 has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2088083343911424000","occurred_at":"2026-08-14T02:00:38Z","signature_id":"7ccfe3ca4663a367e94b3f21db35cbc73f35125a03c66dc4868c1e5fd82b70f1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"aef1ba5dc0db05ae425be4ca4f1d6c3019b64b24c464bcaee15867c943f90b24","failure_pattern":"VerifyVMOSDiskCustomerEncryption(nodePool=\u003cnodepool\u003e) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-w9d2c-bm8tg_OSDisk for VM des-encrypt-des-np-w9d2c-bm8tg has encryption type EncryptionAtRestWithPlatformKey, expected","search_query":"VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-w9d2c-bm8tg_OSDisk for VM des-encrypt-des-np-w9d2c-bm8tg has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-08..2026-08-14 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:163]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc000d9adf8\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-w9d2c-bm8tg_OSDisk for VM des-encrypt-des-np-w9d2c-bm8tg has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    disk des-encrypt-des-np-w9d2c-djq6l_OSDisk for VM des-encrypt-des-np-w9d2c-djq6l has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:163]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc000d9adf8\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-w9d2c-bm8tg_OSDisk for VM des-encrypt-des-np-w9d2c-bm8tg has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    disk des-encrypt-des-np-w9d2c-djq6l_OSDisk for VM des-encrypt-des-np-w9d2c-djq6l has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2088400490290221056","occurred_at":"2026-08-14T23:00:52Z","signature_id":"dc73246ae54a0549a7dcd7f95ed53701d177b10cf154541ca3c6bd4dd79ec73a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":30,"trend_counts":[0,0,0,0,4,4,1],"trend_range":"2026-08-08..2026-08-14 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-20","2026-07-27","2026-08-03"],"prior_runs_affected":30,"prior_last_seen_at":"2026-08-05T23:00:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2088087349807288320","occurred_at":"2026-08-14T02:16:33Z","signature_id":"a8557b42abcd154be2dbc1d4e7697becab9013bfd159699fe454c3dbe99a6090","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}