{"meta":{"start_date":"2026-06-29","end_date":"2026-07-05","start_at":"2026-06-29T00:00:00Z","end_at":"2026-07-06T00:00:00Z","timezone":"UTC","generated_at":"2026-08-27T07:05:04Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":14,"failed_runs":5,"raw_occurrences":11,"matched_occurrences":11,"runs_affected":5},"rows":[{"environment":"prod","failure_pattern_id":"7aebc21de77cb64a5180833ea520c7de32315e635ee94e975fa51f1b8f3df2fd","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.fips-enabled\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":28.571428571428573,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,2,2],"trend_range":"2026-06-28..2026-07-05 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008ee580\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-62994n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-62994n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.fips-enabled\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.fips-enabled]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008ee580\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-62994n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-62994n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.fips-enabled\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.fips-enabled]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001524b60\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-xfhbxl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-xfhbxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.fips-enabled\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.fips-enabled]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001524b60\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-xfhbxl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-xfhbxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.fips-enabled\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.fips-enabled]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e8f8e0\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bd426p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-bd426p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.fips-enabled\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.fips-enabled]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e8f8e0\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bd426p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-bd426p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.fips-enabled\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.fips-enabled]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2073904828865581056","occurred_at":"2026-07-05T23:00:17Z","signature_id":"bf02e674ee0140b57bfa646d752278339abd9611066b94dea9da66724afca8db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2073587735230156800","occurred_at":"2026-07-05T02:00:16Z","signature_id":"bcc522c21bb2cde4c9b58f2516e179d61f29634d7503424351c13f77dafcc31f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2073542435853570048","occurred_at":"2026-07-04T23:00:15Z","signature_id":"a64d9f1b4a520e381c9c6d5527f7a523f7aa66d590c4451b062019fd9d7ee240","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2073225336216096768","occurred_at":"2026-07-04T02:00:13Z","signature_id":"8dfe488779a58d8c72653e934e8d7169cd70d98d2b8b68e7d868b61558a3352d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"7ac9ea1609c23f13fc59d25bc0226d6ddbb91e9217b1f67c5624f53eb77e8333","failure_pattern":"error should indicate FIPS value must be true or false","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should reject an HCP cluster creation with invalid FIPS tag value","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":28.571428571428573,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,2,2],"trend_range":"2026-06-28..2026-07-05 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should reject an HCP cluster creation with invalid FIPS tag value","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should reject an HCP cluster creation with invalid FIPS tag value","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2073904828865581056","occurred_at":"2026-07-05T23:00:17Z","signature_id":"de8314c475995e0785e91e95c202ce3fffed3fe3c3aef8af44fc920a56292561","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2073587735230156800","occurred_at":"2026-07-05T02:00:16Z","signature_id":"de8314c475995e0785e91e95c202ce3fffed3fe3c3aef8af44fc920a56292561","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2073542435853570048","occurred_at":"2026-07-04T23:00:15Z","signature_id":"de8314c475995e0785e91e95c202ce3fffed3fe3c3aef8af44fc920a56292561","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2073225336216096768","occurred_at":"2026-07-04T02:00:13Z","signature_id":"de8314c475995e0785e91e95c202ce3fffed3fe3c3aef8af44fc920a56292561","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"637f5d3cd55d4f747d26b4df077f4afc1c471d422a13cce2120c0226a485046b","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.142857142857143,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-06-28..2026-07-05 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:88]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008b3380\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-autoscaling-hcp-cluster-zznhms\" in resourcegroup=\"autoscaling-cluster-jzz264\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/autoscaling-cluster-jzz264/providers/Microsoft.Resources/deployments/customer-infra-autoscaling-hcp-cluster-zznhms/operationStatuses/08584188326625955293\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-ljrmjtkjzfdwc' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:88]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008b3380\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-autoscaling-hcp-cluster-zznhms\" in resourcegroup=\"autoscaling-cluster-jzz264\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/autoscaling-cluster-jzz264/providers/Microsoft.Resources/deployments/customer-infra-autoscaling-hcp-cluster-zznhms/operationStatuses/08584188326625955293\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-ljrmjtkjzfdwc' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2071730535012503552","occurred_at":"2026-06-29T23:00:25Z","signature_id":"271c5d33f1a61fb455d97690f43152ab34cbede05a8027ae033181ca794977fe","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-85j8vv, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-85j8vv in resource group rg-candidate-4-21-qjdhn9, error: failed waiting for cluster=\"cluster-candidate-4-21-85j8vv\" in resourcegroup=\"rg-candidate-4-21-qjdhn9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-85j8vv in resource group rg-candidate-4-21-qjdhn9, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.142857142857143,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-06-28..2026-07-05 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-01","2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":62,"prior_last_seen_at":"2026-06-25T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-qjdhn9/cluster-candidate-4-21-85j8vv should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00139fda0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-85j8vv, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-85j8vv in resource group rg-candidate-4-21-qjdhn9, error: failed waiting for cluster=\"cluster-candidate-4-21-85j8vv\" in resourcegroup=\"rg-candidate-4-21-qjdhn9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-85j8vv in resource group rg-candidate-4-21-qjdhn9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-qjdhn9/cluster-candidate-4-21-85j8vv should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00139fda0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-85j8vv, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-85j8vv in resource group rg-candidate-4-21-qjdhn9, error: failed waiting for cluster=\"cluster-candidate-4-21-85j8vv\" in resourcegroup=\"rg-candidate-4-21-qjdhn9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-85j8vv in resource group rg-candidate-4-21-qjdhn9, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2073225336216096768","occurred_at":"2026-07-04T02:00:13Z","signature_id":"0be599c110dcfa90eeb78743877d61b7bed641a33efb9f313dbb477ef78c02b8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.142857142857143,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-06-28..2026-07-05 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-01","2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":8,"prior_last_seen_at":"2026-06-23T01:55:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001196390\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-2c5dq.apps.aro.cilium-cluster.kekw.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001196390\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-2c5dq.apps.aro.cilium-cluster.kekw.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2073587735230156800","occurred_at":"2026-07-05T02:00:16Z","signature_id":"96def55c5b0763a240e2aee3ebdffda425fa7de5c5ec6bd38ebdaae12cb965f8","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}